Commit 8542e9435 for llama.cpp
commit 8542e94350f0ade01644db6f1d1520da3814dafd
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Tue Oct 27 14:44:41 2026 +0200
tests : speed up tensor data generation and model saving (#31145)
* tests : speed up tensor data generation and model saving
Refactor save_models to create only the model, not a llama_context,
which is not needed for saving. Replace the slow std::normal_distribution
draw with a splitmix64 + box-muller normal generator.
Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* tests : use a shared pool of normal draws for tensor data
Generate a pool of normal draws once with std::mt19937 and
std::normal_distribution, then have every tensor copy a cyclic slice of
it instead of running a per-tensor box-muller. The name-keyed slice
selection keeps the draw deterministic.
Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* tests : log the size of saved models
Report the produced file size in bytes and MB after each save.
Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
* tests : simplify the model save log
Put the model name, target and size on one line per saved model.
Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 8b8ab6e5d..52d843f51 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -8,6 +8,7 @@
#include "llama-cpp.h"
#include "../src/llama-ext.h"
+#include <algorithm>
#include <cinttypes>
#include <cmath>
#include <cstddef>
@@ -54,12 +55,34 @@ struct tensor_data_params {
// fills the values of one tensor, the name carries whatever the property needs
typedef void (*tensor_filler)(std::vector<float> & dst, const char * name, const tensor_data_params & params);
+// a pool of normal draws, generated once and reused for every model
+static std::vector<float> g_normal_pool;
+static const size_t NORMAL_POOL_SIZE = 16 * 1024 * 1024;
+
+// fill the pool once; each tensor copies a slice of it
+static void init_normal_pool(const size_t seed, const float stdev) {
+ g_normal_pool.resize(NORMAL_POOL_SIZE);
+ std::mt19937 gen(seed);
+ std::normal_distribution<float> dist(0.0f, stdev);
+ for (float & v : g_normal_pool) {
+ v = dist(gen);
+ }
+}
+
// a normal draw keyed on a name, so that tensors which share a draw ask for the same key
-static void fill_normal(std::vector<float> & dst, const char * key, const tensor_data_params & params) {
- std::mt19937 gen(params.seed ^ std::hash<std::string>{}(key));
- std::normal_distribution<float> dis(0.0f, params.stdev);
- for (float & v : dst) {
- v = dis(gen);
+static void fill_normal(std::vector<float> & dst, const char * key, const tensor_data_params &) {
+ const size_t n = dst.size();
+ const size_t pool_size = g_normal_pool.size();
+ const size_t start = std::hash<std::string>{}(key) % pool_size;
+
+ // copy a cyclic slice of the pool, so a tensor larger than the pool still fills
+ size_t i = 0;
+ size_t pos = start;
+ while (i < n) {
+ const size_t chunk = std::min(pool_size - pos, n - i);
+ std::memcpy(dst.data() + i, g_normal_pool.data() + pos, chunk * sizeof(float));
+ i += chunk;
+ pos = 0;
}
}
@@ -682,11 +705,11 @@ static std::string get_tmp_gguf_path() {
return (std::filesystem::temp_directory_path() / ("test-llama-archs-" + std::to_string(++id) + ".gguf")).string();
}
-static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
+static llama_model_ptr get_model(
struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const float stdev,
const std::vector<ggml_backend_dev_t> & devs,
- const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false,
- const llama_model_tensor_buft_override * tensor_buft_overrides = nullptr, const size_t moe_cache_size = 0) {
+ const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER,
+ const llama_model_tensor_buft_override * tensor_buft_overrides = nullptr) {
GGML_ASSERT((gguf_ctx == nullptr) != (file == nullptr));
llama_model_params model_params = llama_model_default_params();
model_params.progress_callback = silent_model_load_progress;
@@ -696,6 +719,23 @@ static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
model_params.split_mode = split_mode;
model_params.tensor_buft_overrides = tensor_buft_overrides;
+ tensor_data_params tensor_params = { seed, stdev };
+ llama_model_ptr model(gguf_ctx != nullptr ?
+ llama_model_init_from_user(gguf_ctx, set_tensor_data, &tensor_params, model_params) :
+ llama_model_load_from_file_ptr(file, model_params));
+ if (!model) {
+ throw std::runtime_error("failed to create llama model");
+ }
+ return model;
+}
+
+static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
+ struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const float stdev,
+ const std::vector<ggml_backend_dev_t> & devs,
+ const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false,
+ const llama_model_tensor_buft_override * tensor_buft_overrides = nullptr, const size_t moe_cache_size = 0) {
+ llama_model_ptr model = get_model(gguf_ctx, file, seed, stdev, devs, split_mode, tensor_buft_overrides);
+
llama_context_params ctx_params = llama_context_default_params();
ctx_params.n_ctx = 0;
ctx_params.n_threads = 4;
@@ -709,13 +749,6 @@ static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
ctx_params.n_ubatch = 2;
}
- tensor_data_params tensor_params = { seed, stdev };
- llama_model_ptr model(gguf_ctx != nullptr ?
- llama_model_init_from_user(gguf_ctx, set_tensor_data, &tensor_params, model_params) :
- llama_model_load_from_file_ptr(file, model_params));
- if (!model) {
- throw std::runtime_error("failed to create llama model");
- }
llama_context_ptr lctx(llama_init_from_model(model.get(), ctx_params));
if (!lctx) {
throw std::runtime_error("failed to create llama context");
@@ -1027,10 +1060,12 @@ static int save_models(const std::string & arch_filter, const size_t seed, const
continue;
}
gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe);
- auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {});
+ llama_model_ptr model = get_model(gguf_ctx.get(), nullptr, seed, stdev, {});
const std::string path = dir + "/" + llama_arch_name(arch) + (moe ? "-moe.gguf" : "-dense.gguf");
- LOG_INF("%s: Saving %s model (%s) to %s...\n", __func__, llama_arch_name(arch), moe ? "MoE" : "dense", path.c_str());
- llama_model_save_to_file(model_and_ctx.first.get(), path.c_str());
+ LOG_INF("%s: '%s %s' -> %s ...", __func__, llama_arch_name(arch), moe ? "MoE" : "dense", path.c_str());
+ llama_model_save_to_file(model.get(), path.c_str());
+ const std::uintmax_t size = std::filesystem::file_size(path);
+ LOG_CNT(" (%.2f MB)\n", (double) size / (1024.0 * 1024.0));
}
}
llama_log_set(ud.log_old.callback, ud.log_old.user_data);
@@ -1419,6 +1454,8 @@ int main(int argc, char ** argv) {
}
LOG_INF("%s: using seed %zu, stdev %f\n", __func__, seed, stdev);
+ init_normal_pool(seed, stdev);
+
try {
if (!out.empty()) {
return save_models(arch_filter, seed, stdev, verbosity, out);