Commit 161755f29 for llama.cpp
commit 161755f29e415e2c33efe906e91843c068efd664
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Mon Sep 21 13:57:36 2026 +0300
test-llama-archs : make tensor data stdev configurable and improve help (#29133)
* test-llama-archs : make tensor data stdev configurable
Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp
* test-llama-archs : expand usage and add examples
Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp
* test-llama-archs : fail on unknown args and log usage
Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp
* test-llama-archs : add test run summary
Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp
* test-llama-archs : initialize Mamba ssm_a negative
Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp
* test-recurrent-state-rollback : report NMSE for logits mismatches
Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp
* test-recurrent-state-rollback : use NMSE for rollback logits checks
Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp
* tests : disable invalid test
* cont : adjust nmse_eps
* tests : zero DSA indexer score projection in synthetic fixtures
Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp
* cont : add support for `--arch` regex
* cont : alternative top-k stability
* cont : indentation
* cont : fix top-k value
* cont : consistent logs
diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp
index f3b64b210..53caca1a5 100644
--- a/src/models/minimax-m3.cpp
+++ b/src/models/minimax-m3.cpp
@@ -23,7 +23,12 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);
ml.get_key(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, hparams.indexer_block_size);
ml.get_key(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks);
- msa_p = { (int) hparams.indexer_block_size, (int) hparams.indexer_top_k, (int) hparams.indexer_local_blocks };
+
+ msa_p = {
+ /*.blk =*/ (int) hparams.indexer_block_size,
+ /*.topk_blocks =*/ (int) hparams.indexer_top_k,
+ /*.local =*/ (int) hparams.indexer_local_blocks,
+ };
GGML_ASSERT(hparams.indexer_block_size > 0); // avoid div by zero
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 80a045185..488770297 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -12,16 +12,25 @@
#include "../src/llama-model-saver.h"
#include <cinttypes>
+#include <cmath>
#include <cstddef>
#include <cstdio>
#include <cstring>
#include <cstdint>
#include <random>
+#include <regex>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
+static bool arch_matches(const std::string & filter, llm_arch arch) {
+ if (filter.empty()) {
+ return true;
+ }
+ return std::regex_search(llm_arch_name(arch), std::regex(filter));
+}
+
// normalized mean squared error = mse(a, b) / mse(a, 0)
static double nmse(const std::vector<float> & a, const std::vector<float> & b) {
GGML_ASSERT(a.size() == b.size());
@@ -39,24 +48,36 @@ static double nmse(const std::vector<float> & a, const std::vector<float> & b) {
return mse_a_b / mse_a_0;
}
+struct tensor_data_params {
+ size_t seed;
+ float stdev;
+};
+
static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
- size_t seed = *(const size_t *) userdata;
+ const tensor_data_params & params = *(const tensor_data_params *) userdata;
+ size_t seed = params.seed;
std::hash<std::string> hasher;
seed ^= hasher(tensor->name);
std::mt19937 gen(seed);
- std::normal_distribution<float> dis(0.0f, 1.0e-2f);
+ std::normal_distribution<float> dis(0.0f, params.stdev);
+
+ // TODO: refactor per-tensor initialization logic in a cleaner way
+ // note: Mamba A must be negative (state decay)
+ const bool is_ssm_a = strstr(tensor->name, "ssm_a") != nullptr;
const int64_t ne = ggml_nelements(tensor);
if (tensor->type == GGML_TYPE_F32) {
std::vector<float> tmp(ne);
for (int64_t i = 0; i < ne; i++) {
- tmp[i] = dis(gen);
+ float val = dis(gen);
+ tmp[i] = is_ssm_a ? -fabsf(val) : val;
}
ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));
} else if (tensor->type == GGML_TYPE_F16) {
std::vector<ggml_fp16_t> tmp(ne);
for (int64_t i = 0; i < ne; i++) {
- tmp[i] = ggml_fp32_to_fp16(dis(gen));
+ float val = dis(gen);
+ tmp[i] = ggml_fp32_to_fp16(is_ssm_a ? -fabsf(val) : val);
}
ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));
} else {
@@ -65,7 +86,19 @@ static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
}
static void usage(char ** argv) {
- printf("Usage: %s [-a/--arch arch] [-s/--seed seed] [-o/--out dir] [-v N] [-h/--help]\n", argv[0]);
+ LOG("Usage: %s [options]\n\n", argv[0]);
+ LOG("Options:\n");
+ LOG(" -a, --arch <arch|regex> Run only matching LLM architectures (default: all supported)\n");
+ LOG(" -s, --seed <seed> Set the random seed for tensor initialization and token generation\n");
+ LOG(" -d, --stdev <stdev> Set the standard deviation of the tensor initialization distribution (default: 0.1f)\n");
+ LOG(" -o, --out <dir> Save generated test models to <dir> instead of running backend tests\n");
+ LOG(" -v <N> Set log verbosity level\n");
+ LOG(" -h, --help Show this help message\n\n");
+ LOG("Examples:\n");
+ LOG(" %s\n", argv[0]);
+ LOG(" %s -a qwen35moe\n", argv[0]);
+ LOG(" %s -a deepseek4 -o tests/test-models/\n", argv[0]);
+ LOG(" %s -a cohere2moe -v 5\n", argv[0]);
}
static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed){
@@ -292,7 +325,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,
arch == LLM_ARCH_QWEN4EXP ? n_embd_head : uint32_t(128));
- ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));
+ // note: using a realistic top-k here makes the results unstable and hard to match between CPU and GPU
+ // a large value makes things deterministic since all data is selected by the indexer
+ //ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));
+ ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(131072));
+
ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1));
ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4}));
@@ -314,13 +351,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
}
if (arch == LLM_ARCH_DEEPSEEK4) {
- ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8));
- ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32));
- ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 4, 128}));
- ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f);
- ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
- ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));
- ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
+ ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8));
+ ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32));
+ ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 4, 128}));
+ ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f);
+ ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
+ ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));
+ ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
ms.add_kv(LLM_KV_HASH_LAYER_COUNT, uint32_t(0));
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
@@ -381,19 +418,19 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head);
ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2));
ms.add_kv(LLM_KV_KDA_HEAD_DIM, uint32_t(128));
- ms.add_kv(LLM_KV_KDA_SAFE_GATE, true);
- ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
+ ms.add_kv(LLM_KV_KDA_SAFE_GATE, true);
+ ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
if (arch == LLM_ARCH_BAILINGMOE3) {
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>({0.0f, 4.0f}));
ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>({0.0f, 5.0f}));
}
- ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);
- ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));
- ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);
- ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12));
- ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f);
+ ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);
+ ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));
+ ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);
+ ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12));
+ ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f);
ms.add_kv(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, 25.0f);
- ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
+ ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
for (uint32_t il = 0; il < n_layer; il++) {
ggml_tensor t;
@@ -416,7 +453,8 @@ static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/)
}
static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
- struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const std::vector<ggml_backend_dev_t> & devs,
+ struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const float stdev,
+ const std::vector<ggml_backend_dev_t> & devs,
const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false) {
GGML_ASSERT((gguf_ctx == nullptr) != (file == nullptr));
llama_model_params model_params = llama_model_default_params();
@@ -434,9 +472,9 @@ static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
ctx_params.n_ubatch = 64;
}
- size_t tmp = seed;
+ tensor_data_params tensor_params = { seed, stdev };
llama_model_ptr model(gguf_ctx != nullptr ?
- llama_model_init_from_user(gguf_ctx, set_tensor_data, &tmp, model_params) :
+ llama_model_init_from_user(gguf_ctx, set_tensor_data, &tensor_params, model_params) :
llama_model_load_from_file_ptr(file, model_params));
if (!model) {
throw std::runtime_error("failed to create llama model");
@@ -608,7 +646,7 @@ static bool arch_supported(const llm_arch arch) {
return true;
}
-static int save_models(const llm_arch target_arch, const size_t seed, const int verbosity, const std::string & dir) {
+static int save_models(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity, const std::string & dir) {
struct user_data_t {
struct {
ggml_log_callback callback;
@@ -635,7 +673,7 @@ static int save_models(const llm_arch target_arch, const size_t seed, const int
if (arch == LLM_ARCH_UNKNOWN) {
continue;
}
- if (target_arch != LLM_ARCH_UNKNOWN && arch != target_arch) {
+ if (!arch_matches(arch_filter, arch)) {
continue;
}
if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
@@ -656,7 +694,7 @@ static int save_models(const llm_arch target_arch, const size_t seed, const int
continue;
}
gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe);
- auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {});
+ auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {});
const std::string path = dir + "/" + llm_arch_name(arch) + (moe ? "-moe.gguf" : "-dense.gguf");
LOG_INF("%s: Saving %s model (%s) to %s...\n", __func__, llm_arch_name(arch), moe ? "MoE" : "dense", path.c_str());
llama_model_save_to_file(model_and_ctx.first.get(), path.c_str());
@@ -666,7 +704,7 @@ static int save_models(const llm_arch target_arch, const size_t seed, const int
return 0;
}
-static int test_backends(const llm_arch target_arch, const size_t seed, const int verbosity) {
+static int test_backends(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity) {
struct user_data_t {
struct {
ggml_log_callback callback;
@@ -731,22 +769,24 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
const std::string template_row_res = "%15s %10s|%20s|\n";
bool all_ok = true;
+ size_t n_tests = 0;
+ size_t n_failed = 0;
common_log_flush(common_log_main());
- printf(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip");
- printf("|");
+ LOG(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip");
+ LOG("|");
for (size_t i = 0; i < max_arch_name_length; i++) {
- printf("-");
+ LOG("-");
}
- printf("|");
+ LOG("|");
for (size_t i = 0; i < max_device_label_length; i++) {
- printf("-");
+ LOG("-");
}
- printf("|------|---------------|---------|\n");
+ LOG("|------|---------------|---------|\n");
for (const llm_arch & arch : llm_arch_all()) {
if (arch == LLM_ARCH_UNKNOWN) {
continue;
}
- if (target_arch != LLM_ARCH_UNKNOWN && arch != target_arch) {
+ if (!arch_matches(arch_filter, arch)) {
continue;
}
if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
@@ -773,8 +813,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
std::vector<float> logits_cpu;
for (device_config & dc : dev_configs) {
// print test config first; should anything fail during model loading or inference, at least we know which test case caused it
- printf(template_row_cfg.c_str(),
- llm_arch_name(arch), dc.label.c_str(), config_name.c_str());
+ LOG(template_row_cfg.c_str(), llm_arch_name(arch), dc.label.c_str(), config_name.c_str());
fflush(stdout);
std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_dev;
@@ -784,19 +823,22 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
char nmse_str[12] = {0};
bool skip = !arch_supported(arch) || (dc.split_mode == LLAMA_SPLIT_MODE_TENSOR && dc.devs.empty());
+ bool test_executed = false;
+ bool test_ok = true;
if (!skip) {
if (logits_cpu.empty()) {
- model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {}, LLAMA_SPLIT_MODE_LAYER, encode);
+ model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {}, LLAMA_SPLIT_MODE_LAYER, encode);
logits_cpu = get_logits(model_and_ctx_cpu.first.get(), model_and_ctx_cpu.second.get(), tokens, encode);
}
if (dc.split_mode != LLAMA_SPLIT_MODE_TENSOR || llm_arch_supports_sm_tensor(arch)) {
- model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, dc.devs, dc.split_mode, encode);
+ test_executed = true;
+ model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, dc.devs, dc.split_mode, encode);
logits_dev = get_logits(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens, encode);
const double nmse_val = nmse(logits_cpu, logits_dev);
snprintf(nmse_str, sizeof(nmse_str), "(%.2e)", nmse_val);
status_nmse = "\033[1;32mOK\033[0m";
if (nmse_val > 1e-4) {
- all_ok = false;
+ test_ok = false;
status_nmse = "\033[1;31mFAIL\033[0m";
}
}
@@ -805,6 +847,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
// FIXME: when adding a tensor to a gguf_context a copy is made, this changes the pointer which the meta backend
// in turn uses to map the tensors to their simple equivalents - this is fundamentally incompatible
if (file != nullptr && llama_model_saver_supports_arch(arch) && dc.split_mode != LLAMA_SPLIT_MODE_TENSOR) {
+ test_executed = true;
GGML_ASSERT(model_and_ctx_dev.first && model_and_ctx_dev.second);
llama_model_saver ms = llama_model_saver(model_and_ctx_dev.first.get());
ms.add_kv_from_model();
@@ -812,14 +855,14 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
ms.save(file);
rewind(file);
- auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, dc.devs, dc.split_mode, encode);
+ auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, stdev, dc.devs, dc.split_mode, encode);
const std::vector<float> logits_roundtrip = get_logits(
model_and_ctx_roundtrip.first.get(), model_and_ctx_roundtrip.second.get(), tokens, encode);
status_roundtrip = "\033[1;32mOK\033[0m";
GGML_ASSERT(logits_roundtrip.size() == logits_dev.size());
for (size_t i = 0; i < logits_roundtrip.size(); i++) {
if (logits_roundtrip[i] != logits_dev[i]) {
- all_ok = false;
+ test_ok = false;
status_roundtrip = "\033[1;31mFAIL\033[0m";
break;
}
@@ -827,12 +870,28 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
}
}
+ if (test_executed) {
+ n_tests++;
+ if (!test_ok) {
+ n_failed++;
+ all_ok = false;
+ }
+ }
+
// log the results for this test case
- printf(template_row_res.c_str(),
- status_nmse.c_str(), nmse_str, status_roundtrip.c_str());
+ LOG(template_row_res.c_str(), status_nmse.c_str(), nmse_str, status_roundtrip.c_str());
}
}
}
+
+ if (n_tests == 0) {
+ LOG("Summary: no tests executed\n");
+ } else if (n_failed == 0) {
+ LOG("Summary: all %zu test(s) passed\n", n_tests);
+ } else {
+ LOG("Summary: %zu test(s) executed, %zu failed\n", n_tests, n_failed);
+ }
+
llama_log_set(ud.log_old.callback, ud.log_old.user_data);
return all_ok ? 0 : 1;
}
@@ -844,8 +903,9 @@ int main(int argc, char ** argv) {
std::random_device rd;
- llm_arch arch = LLM_ARCH_UNKNOWN;
+ std::string arch_filter;
size_t seed = rd();
+ float stdev = 0.1f;
std::string out;
int verbosity = LOG_LEVEL_ERROR;
@@ -854,52 +914,70 @@ int main(int argc, char ** argv) {
if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
usage(argv);
return 0;
- }
- if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {
+ } else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {
if (i + 1 < argc) {
const std::string arch_name = argv[++i];
- arch = llm_arch_from_string(arch_name);
- if (arch == LLM_ARCH_UNKNOWN) {
- LOG_ERR("%s: unkown LLM architecture: %s\n", __func__, arch_name.c_str());
- return 1;
+ if (llm_arch_from_string(arch_name) != LLM_ARCH_UNKNOWN) {
+ // exact architecture name
+ arch_filter = "^" + arch_name + "$";
+ } else {
+ try {
+ std::regex re(arch_name);
+ arch_filter = arch_name;
+ } catch (const std::regex_error & err) {
+ LOG_ERR("%s: invalid architecture regex: %s (%s)\n", __func__, arch_name.c_str(), err.what());
+ return 1;
+ }
}
} else {
usage(argv);
return 1;
}
- }
- if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) {
+ } else if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) {
if (i + 1 < argc) {
seed = std::stoull(argv[++i]);
} else {
usage(argv);
return 1;
}
- }
- if (strcmp(argv[i], "-v") == 0) {
+ } else if (strcmp(argv[i], "-d") == 0 || strcmp(argv[i], "--stdev") == 0) {
+ if (i + 1 < argc) {
+ stdev = std::stof(argv[++i]);
+ } else {
+ usage(argv);
+ return 1;
+ }
+ } else if (strcmp(argv[i], "-v") == 0) {
if (i + 1 < argc) {
verbosity = std::stoull(argv[++i]);
} else {
usage(argv);
return 1;
}
- }
- if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) {
+ } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) {
if (i + 1 < argc) {
out = argv[++i];
} else {
usage(argv);
return 1;
}
+ } else {
+ LOG_ERR("%s: unknown argument: %s\n", __func__, argv[i]);
+ usage(argv);
+ return 1;
}
}
- printf("%s: using seed %zu\n", __func__, seed);
+ if (stdev <= 0.0f) {
+ LOG_ERR("%s: stdev must be > 0\n", __func__);
+ return 1;
+ }
+ LOG_INF("%s: using seed %zu, stdev %f\n", __func__, seed, stdev);
try {
if (!out.empty()) {
- return save_models(arch, seed, verbosity, out);
+ return save_models(arch_filter, seed, stdev, verbosity, out);
}
- return test_backends(arch, seed, verbosity);
+ return test_backends(arch_filter, seed, stdev, verbosity);
} catch (const std::exception & err) {
fprintf(stderr, "encountered runtime error: %s\n", err.what());
return -1;
diff --git a/tests/test-recurrent-state-rollback.cpp b/tests/test-recurrent-state-rollback.cpp
index ef05de67d..fdac344d8 100644
--- a/tests/test-recurrent-state-rollback.cpp
+++ b/tests/test-recurrent-state-rollback.cpp
@@ -90,6 +90,20 @@ static float logit_diff(float a, float b) {
return std::isfinite(a) && std::isfinite(b) ? std::fabs(a - b) : std::numeric_limits<float>::infinity();
}
+static double nmse(const float * a, const float * b, int n) {
+ double mse_ab = 0.0;
+ double mse_a0 = 0.0;
+ for (int i = 0; i < n; i++) {
+ if (!std::isfinite(a[i]) || !std::isfinite(b[i])) {
+ return std::numeric_limits<double>::infinity();
+ }
+ const double diff = (double) a[i] - b[i];
+ mse_ab += diff*diff;
+ mse_a0 += (double) a[i]*a[i];
+ }
+ return mse_a0 == 0.0 ? (mse_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : mse_ab/mse_a0;
+}
+
// Roll back multiple sequences, then replay them in a single batch whose
// per-seq token count exceeds n_ubatch: each seq's replay spans several
// ubatches while its rollback restore is still pending. Compared against a
@@ -182,13 +196,15 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
return false;
}
- // identical ubatch shapes from bit-exact states: a correct implementation
- // matches bitwise, so eps only allows backend scheduling noise
- constexpr float eps = 1e-7f;
+ // identical ubatch shapes should produce identical states, but the larger
+ // stdev makes the model sensitive to backend scheduling/rounding noise
+ constexpr float nmse_eps = 1e-5f;
float diff_max = 0.0f;
uint32_t seq_first = 0;
int32_t pos_first = -1;
+ double nmse_ab = 0.0;
+ double nmse_a0 = 0.0;
for (uint32_t i = 0; i < n_seqs*n_replay; ++i) {
const float * l_roll = llama_get_logits_ith(ctx_roll, i);
const float * l_ref = llama_get_logits_ith(ctx_ref, i);
@@ -198,23 +214,34 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
return false;
}
for (int t = 0; t < n_vocab; ++t) {
- const float diff = logit_diff(l_roll[t], l_ref[t]);
- if (diff > eps && pos_first < 0) {
+ const float r = l_roll[t];
+ const float f = l_ref[t];
+ const float diff = logit_diff(r, f);
+ if (diff > 0.0f && pos_first < 0) {
seq_first = i/n_replay;
pos_first = p0 + (int32_t) (i%n_replay);
}
diff_max = std::max(diff_max, diff);
+ if (std::isfinite(r) && std::isfinite(f)) {
+ const double d = (double) r - f;
+ nmse_ab += d*d;
+ nmse_a0 += (double) r*r;
+ } else {
+ nmse_ab = std::numeric_limits<double>::infinity();
+ nmse_a0 = 1.0;
+ }
}
}
+ const double nmse_val = nmse_a0 == 0.0 ? (nmse_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : nmse_ab/nmse_a0;
- if (diff_max > eps) {
- fprintf(stderr, "%s : multi-seq split replay logits mismatch (max diff %g, first at seq %u pos %d)\n",
- __func__, (double) diff_max, seq_first, pos_first);
+ if (nmse_val > nmse_eps) {
+ fprintf(stderr, "%s : multi-seq split replay logits mismatch (max diff %g, nmse %g, first at seq %u pos %d)\n",
+ __func__, (double) diff_max, nmse_val, seq_first, pos_first);
cleanup();
return false;
}
- fprintf(stderr, "%s : multi-seq split replay matched (max diff %g)\n", __func__, (double) diff_max);
+ fprintf(stderr, "%s : multi-seq split replay matched (max diff %g, nmse %g)\n", __func__, (double) diff_max, nmse_val);
// seq-1-only decodes must be independent of seq 0's content: diverge seq 0
// in ctx_ref only, then compare identical seq-1-only continuations bitwise
@@ -231,6 +258,8 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
}
float diff_tail = 0.0f;
+ double nmse_tail_ab = 0.0;
+ double nmse_tail_a0 = 0.0;
for (uint32_t i = 0; i < n_tail && ok; ++i) {
const llama_pos pos = p0 + (llama_pos) (n_replay + i);
llama_batch batch_one = llama_batch_init(1, 0, 1);
@@ -246,18 +275,29 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
const float * l_ref = llama_get_logits_ith(ctx_ref, 0);
ok = l_roll != nullptr && l_ref != nullptr;
for (int t = 0; ok && t < n_vocab; ++t) {
- diff_tail = std::max(diff_tail, logit_diff(l_roll[t], l_ref[t]));
+ const float r = l_roll[t];
+ const float f = l_ref[t];
+ diff_tail = std::max(diff_tail, logit_diff(r, f));
+ if (std::isfinite(r) && std::isfinite(f)) {
+ const double d = (double) r - f;
+ nmse_tail_ab += d*d;
+ nmse_tail_a0 += (double) r*r;
+ } else {
+ nmse_tail_ab = std::numeric_limits<double>::infinity();
+ nmse_tail_a0 = 1.0;
+ }
}
}
+ const double nmse_tail = nmse_tail_a0 == 0.0 ? (nmse_tail_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : nmse_tail_ab/nmse_tail_a0;
- if (!ok || diff_tail > eps) {
- fprintf(stderr, "%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g)\n",
- __func__, ok ? 1 : 0, (double) diff_tail);
+ if (!ok || nmse_tail > nmse_eps) {
+ fprintf(stderr, "%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g, nmse %g)\n",
+ __func__, ok ? 1 : 0, (double) diff_tail, nmse_tail);
cleanup();
return false;
}
- fprintf(stderr, "%s : seq-1-only decode independent of seq 0 (max diff %g)\n", __func__, (double) diff_tail);
+ fprintf(stderr, "%s : seq-1-only decode independent of seq 0 (max diff %g, nmse %g)\n", __func__, (double) diff_tail, nmse_tail);
cleanup();
return true;
}
@@ -266,6 +306,7 @@ static int test_rollback(const common_params & params, llama_model * model, uint
const llama_vocab * vocab = llama_model_get_vocab(model);
const int n_vocab = llama_vocab_n_tokens(vocab);
+ // TODO: use smart pointers
llama_context * ctx_src = make_ctx(params, model, fill);
llama_context * ctx_dst = make_ctx(params, model, fill);
if (ctx_src == nullptr || ctx_dst == nullptr) {
@@ -320,7 +361,7 @@ static int test_rollback(const common_params & params, llama_model * model, uint
ckpt.update_tgt(ctx_src, 0, 0);
ckpt.load_tgt(ctx_dst, 0, 0);
- constexpr float eps = 1e-5f;
+ constexpr float nmse_eps = 0.0;
std::vector<std::vector<float>> logits_src_replay(n_rollback);
const auto replay_and_compare = [&](const char * mode) {
for (uint32_t i = 0; i < n_rollback; ++i) {
@@ -339,13 +380,18 @@ static int test_rollback(const common_params & params, llama_model * model, uint
}
logits_src_replay[i].assign(logits_src, logits_src + n_vocab);
+ const double nmse_val = nmse(logits_src, logits_dst, n_vocab);
+ int token_first = -1;
for (int token = 0; token < n_vocab; ++token) {
- if (logit_diff(logits_src[token], logits_dst[token]) > eps) {
- fprintf(stderr, "%s : %s logits mismatch at position %d, token %d (%g != %g)\n",
- __func__, mode, pos, token, (double) logits_src[token], (double) logits_dst[token]);
- return false;
+ if (logit_diff(logits_src[token], logits_dst[token]) > 0.0f && token_first < 0) {
+ token_first = token;
}
}
+ if (nmse_val > nmse_eps) {
+ fprintf(stderr, "%s : %s logits mismatch at position %d, first token %d, nmse %g\n",
+ __func__, mode, pos, token_first, nmse_val);
+ return false;
+ }
}
return true;
};
@@ -353,20 +399,22 @@ static int test_rollback(const common_params & params, llama_model * model, uint
return 1;
}
- if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1) ||
- !llama_memory_seq_rm(llama_get_memory(ctx_dst), 0, rollback_pos, -1)) {
- fprintf(stderr, "%s : partial rollback failed\n", __func__);
- return 1;
- }
+ // TODO: this test is invalid because RS rollback is only correct once after a ubatch with more than n_rs_seq tokens
+ // this is not the case here. add asserts and guardrails to prevent such attempts
+ //if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1) ||
+ // !llama_memory_seq_rm(llama_get_memory(ctx_dst), 0, rollback_pos, -1)) {
+ // fprintf(stderr, "%s : partial rollback failed\n", __func__);
+ // return 1;
+ //}
- constexpr llama_state_seq_flags partial_flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY;
- common_prompt_checkpoint ckpt_partial;
- ckpt_partial.update_tgt(ctx_src, 0, partial_flags);
- ckpt_partial.load_tgt(ctx_dst, 0, partial_flags);
+ //constexpr llama_state_seq_flags partial_flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY;
+ //common_prompt_checkpoint ckpt_partial;
+ //ckpt_partial.update_tgt(ctx_src, 0, partial_flags);
+ //ckpt_partial.load_tgt(ctx_dst, 0, partial_flags);
- if (!replay_and_compare("partial")) {
- return 1;
- }
+ //if (!replay_and_compare("partial")) {
+ // return 1;
+ //}
// Repeat the load into a context that already has its own rollback state:
// groups 1..n_rs_seq hold a different prompt's history, and rs_idx[0] is
@@ -408,13 +456,18 @@ static int test_rollback(const common_params & params, llama_model * model, uint
return 1;
}
+ const double nmse_dirty = nmse(logits_src_replay[i].data(), logits_dirty, n_vocab);
+ int token_first = -1;
for (int token = 0; token < n_vocab; ++token) {
- if (logit_diff(logits_src_replay[i][token], logits_dirty[token]) > eps) {
- fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, token %d (%g != %g)\n",
- __func__, pos, token, (double) logits_src_replay[i][token], (double) logits_dirty[token]);
- return 1;
+ if (logit_diff(logits_src_replay[i][token], logits_dirty[token]) > 0.0f && token_first < 0) {
+ token_first = token;
}
}
+ if (nmse_dirty > nmse_eps) {
+ fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, first token %d, nmse %g\n",
+ __func__, pos, token_first, nmse_dirty);
+ return 1;
+ }
}
fprintf(stderr, "%s : recurrent rollback checkpoint restored successfully\n", __func__);