Commit 0a7675746 for llama.cpp
commit 0a76757469ea94ad5f88d14e18cc7370b9931f99
Author: mahounairo <24420747+mahounairo@users.noreply.github.com>
Date: Thu Nov 19 00:08:12 2026 +0100
llama : fix sticky NaN in recurrent state (#31913)
Co-authored-by: Alexey Kopytko <alexey@kopytko.com>
diff --git a/src/llama-ext.h b/src/llama-ext.h
index 2b2256fb6..b8b7fba0b 100644
--- a/src/llama-ext.h
+++ b/src/llama-ext.h
@@ -116,6 +116,8 @@ LLAMA_API ggml_backend_dev_t llama_model_get_device(const struct llama_model * m
LLAMA_API llama_memory_breakdown llama_get_memory_breakdown(const struct llama_context * ctx);
+LLAMA_API std::vector<ggml_tensor *> llama_memory_get_tensors(const llama_memory_t mem);
+
// the backend buffers that a state write of this memory would touch - empty caches report none
LLAMA_API std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t mem);
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 7b01af1b9..58c5b6c9c 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -4103,7 +4103,7 @@ ggml_tensor * llm_graph_context::build_rs(
// Clear a single state which will then be copied to the other cleared states.
// Note that this is a no-op when the view is zero-sized.
ggml_tensor * state_zero = ggml_view_1d(ctx0, states, state_size*(rs_zero >= 0), rs_zero*states->nb[1]*(rs_zero >= 0));
- ggml_build_forward_expand(gf, ggml_scale_inplace(ctx0, state_zero, 0));
+ ggml_build_forward_expand(gf, ggml_fill_inplace(ctx0, state_zero, 0.0f));
// copy states
// NOTE: assuming the copy destinations are ALL contained between rs_head and rs_head + n_rs
diff --git a/src/llama-memory.cpp b/src/llama-memory.cpp
index 042aab608..80099e76a 100644
--- a/src/llama-memory.cpp
+++ b/src/llama-memory.cpp
@@ -112,9 +112,9 @@ std::map<ggml_backend_buffer_type_t, size_t> llama_memory_bufs::memory_breakdown
return ret;
}
-// collects the buffers of the tensors that a state write visits, without keeping the data
-struct llama_memory_buffers_writer : public llama_io_write_i {
- std::set<ggml_backend_buffer_t> buffers;
+// collects the tensors that a state write visits, without keeping the data
+struct llama_memory_tensors_writer : public llama_io_write_i {
+ std::set<ggml_tensor *> tensors;
size_t size = 0;
void write(const void *, size_t n) override {
@@ -122,7 +122,7 @@ struct llama_memory_buffers_writer : public llama_io_write_i {
}
void write_tensor(ggml_tensor * tensor, size_t, size_t n) override {
- buffers.insert(tensor->buffer);
+ tensors.insert(tensor);
size += n;
}
@@ -132,13 +132,22 @@ struct llama_memory_buffers_writer : public llama_io_write_i {
};
std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t mem) {
+ std::set<ggml_backend_buffer_t> buffers;
+ for (ggml_tensor * tensor : llama_memory_get_tensors(mem)) {
+ buffers.insert(tensor->buffer);
+ }
+
+ return std::vector<ggml_backend_buffer_t>(buffers.begin(), buffers.end());
+}
+
+std::vector<ggml_tensor *> llama_memory_get_tensors(const llama_memory_t mem) {
if (mem == nullptr) {
return {};
}
- llama_memory_buffers_writer writer;
+ llama_memory_tensors_writer writer;
mem->state_write(writer);
- return std::vector<ggml_backend_buffer_t>(writer.buffers.begin(), writer.buffers.end());
+ return std::vector<ggml_tensor *>(writer.tensors.begin(), writer.tensors.end());
}
diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp
index 0d5d9a469..69ebe5a60 100644
--- a/src/models/deepseek4.cpp
+++ b/src/models/deepseek4.cpp
@@ -200,7 +200,7 @@ static ggml_tensor * dsv4_view_2d(
static ggml_tensor * dsv4_append_zero_row(ggml_context * ctx, ggml_tensor * t, bool neg_inf) {
ggml_tensor * row = ggml_view_1d(ctx, t, t->ne[0], 0);
- row = neg_inf ? ggml_scale_bias(ctx, row, 0.0f, -INFINITY) : ggml_scale(ctx, row, 0.0f);
+ row = ggml_fill(ctx, row, neg_inf ? -INFINITY : 0.0f);
row = ggml_reshape_2d(ctx, row, t->ne[0], 1);
return ggml_concat(ctx, t, row, 1);
diff --git a/tests/llama-context/test-runner.cpp b/tests/llama-context/test-runner.cpp
index 1ca3f4c03..e2d01af4a 100644
--- a/tests/llama-context/test-runner.cpp
+++ b/tests/llama-context/test-runner.cpp
@@ -72,8 +72,9 @@ std::vector<case_run> run_cases(model_run & mr, const std::vector<const ctx_test
} else {
// the recurrent cases are also run against a cache pre-filled with a known
// pattern, so that a restore reading cells it never wrote cannot pass
+ // with 0xff repeated every float of the state becomes NaN to test that the next sequence is not affected
- for (const uint8_t fill : { 0, 0x3e }) {
+ for (const uint8_t fill : { 0, 0x3e, 0xff }) {
TLOG_INF("(fill 0x%02x)\n", fill);
mr.fill = fill;
diff --git a/tests/llama-context/test-util.cpp b/tests/llama-context/test-util.cpp
index b67e2a03b..e95ee7feb 100644
--- a/tests/llama-context/test-util.cpp
+++ b/tests/llama-context/test-util.cpp
@@ -2,7 +2,7 @@
#include "test-log.h"
-#include "../../src/llama-ext.h" // llama_memory_get_buffers, llama_arch_from_string
+#include "../../src/llama-ext.h" // llama_memory_get_buffers, llama_memory_get_tensors, llama_arch_from_string
#include "common.h"
@@ -10,6 +10,7 @@
#include <cstdio>
#include <cstring>
#include <limits>
+#include <regex>
#include <stdexcept>
const double NMSE_THRESHOLD = 1e-5;
@@ -297,6 +298,13 @@ bool corrupt_state(std::vector<uint8_t> & data) {
// context
//
+// the recurrent state tensors: the conv and ssm rows of llama_memory_recurrent and the DeepSeek V4 compressor state
+static bool is_state_tensor(const ggml_tensor * tensor) {
+ static const std::regex re_state("^cache_(r|s|ple_r)_l[0-9]+$|_state_");
+
+ return std::regex_search(tensor->name, re_state);
+}
+
// fill the buffers of the cache before a restore - a restore that reads cells it never wrote
// then produces garbage instead of silently passing
llama_context_ptr init_ctx(llama_model * model, llama_context_params cparams, uint8_t fill) {
@@ -313,11 +321,29 @@ llama_context_ptr init_ctx(llama_model * model, llama_context_params cparams, ui
llama_synchronize(ctx.get());
const auto buffers = llama_memory_get_buffers(llama_get_memory(ctx.get()));
+ const auto tensors = llama_memory_get_tensors(llama_get_memory(ctx.get()));
llama_memory_clear(llama_get_memory(ctx.get()), true);
if (buffers.empty()) {
TLOG_ERR("%s: no cache buffers found\n", __func__);
return nullptr;
}
+
+ // the KV cache only masks the free cells, so NaN there would corrupt the output
+ if (fill == 0xff) {
+ size_t n_state = 0;
+ for (ggml_tensor * tensor : tensors) {
+ if (is_state_tensor(tensor)) {
+ ggml_backend_tensor_memset(tensor, fill, 0, ggml_nbytes(tensor));
+ n_state++;
+ }
+ }
+ if (n_state == 0) {
+ TLOG_ERR("%s: no state tensors found\n", __func__);
+ return nullptr;
+ }
+ return ctx;
+ }
+
for (auto * buffer : buffers) {
ggml_backend_buffer_clear(buffer, fill);
}