Commit 5f490d7bf for llama.cpp
commit 5f490d7bfe78844e6390c869454c149fd7e003a8
Author: Piotr Wilkin (ilintar) <piotr.wilkin@syndatis.com>
Date: Mon Nov 9 19:41:01 2026 +0100
feat: external PLE file loading (#31185)
* model : allow loading external PLE files
Assisted-by: Codex
* convert : add separate PLE exports
Assisted-by: Codex
* tests : fix PLE export test lint
Assisted-by: Codex
* model : make --ple an alias of --shard, allow multiple shard locations
Reuse the multi-shard loading path for external PLE files instead of a
separate option.
* --ple is an alias of --shard: both take a comma separated list and may
be given more than once, and the shards may live anywhere
* metadata comes from the main file; numbered shards are still
auto-discovered and strictly validated
* explicit files outside the discovered set are appended as extra shards,
exempt from split.no and split.count checks
* -isnm/--ignore-shard-num-mismatch loads only the given files, in order,
with no auto-discovery and no split metadata validation
* ple_path, path_ple and llama_model_loader::add_ple_file are removed;
common_model_load() uses llama_model_load_from_splits for two or more
paths
* common/fit.{h,cpp} take the shard path vector, so --fit and the router
memory measurement load every shard
Assisted-by: GPT-6-Astra
* rm -isnm
* make sure to dedup paths
* rm test_convert_ple
* address comments
* allow ple shard to override tensor
---------
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
diff --git a/common/arg.cpp b/common/arg.cpp
index 69be4a61e..81de73cd0 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -3032,6 +3032,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.model.path = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL"));
+ add_opt(common_arg(
+ {"--ple", "--shard"}, "FNAME,...",
+ "extra GGUF shards for the model, comma separated; may be given more than once\n"
+ "paths may live anywhere, including on different drives; --ple is an alias for --shard",
+ [](common_params & params, const std::string & value) {
+ for (const auto & item : parse_csv_row(value)) {
+ params.model.shards.push_back(item);
+ }
+ }
+ ).set_examples({LLAMA_EXAMPLE_COMMON}).set_env("LLAMA_ARG_SHARD"));
add_opt(common_arg(
{"-mu", "--model-url"}, "MODEL_URL",
"model download url (default: unused)",
diff --git a/common/common.cpp b/common/common.cpp
index cca920f09..657dcdbf5 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1258,13 +1258,13 @@ void common_fit_apply(common_params & params, llama_model_params & mparams, llam
cparams_dft.n_rs_seq = 0;
const common_fit_extra_model extra = {
- /*.path_model =*/ params_dft.model.path.c_str(),
+ /*.paths_model =*/ params_dft.model.get_paths(),
/*.mparams =*/ &mparams_dft,
/*.cparams =*/ &cparams_dft,
/*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
};
- common_fit_params(params.model.path.c_str(), &mparams, &cparams,
+ common_fit_params(params.model.get_paths(), &mparams, &cparams,
params.tensor_split,
params.tensor_buft_overrides.data(),
params.fit_params_target.data(),
@@ -1280,7 +1280,7 @@ common_init_result::common_init_result(common_params & params, bool model_only)
common_fit_apply(params, mparams, cparams);
- llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
+ llama_model * model = common_model_load(params.model.get_paths(), mparams);
if (model == NULL) {
return;
}
@@ -1722,6 +1722,19 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
return mparams;
}
+struct llama_model * common_model_load(const std::vector<std::string> & paths, const struct llama_model_params & mparams) {
+ if (paths.size() == 1) {
+ return llama_model_load_from_file(paths[0].c_str(), mparams);
+ }
+
+ std::vector<const char *> c_paths;
+ c_paths.reserve(paths.size());
+ for (const auto & path : paths) {
+ c_paths.push_back(path.c_str());
+ }
+ return llama_model_load_from_splits(c_paths.data(), c_paths.size(), mparams);
+}
+
struct llama_context_params common_context_params_to_llama(const common_params & params) {
auto cparams = llama_context_default_params();
diff --git a/common/common.h b/common/common.h
index 3aae7a8da..cc6eef833 100644
--- a/common/common.h
+++ b/common/common.h
@@ -311,6 +311,18 @@ struct common_params_model {
std::string hf_file = ""; // HF file
std::string docker_repo = ""; // Docker repo
+ std::vector<std::string> shards;
+
+ std::vector<std::string> get_paths() const {
+ std::vector<std::string> paths = { path };
+ for (const auto & shard : shards) {
+ if (std::find(paths.begin(), paths.end(), shard) == paths.end()) {
+ paths.push_back(shard);
+ }
+ }
+ return paths;
+ }
+
std::string get_name() const {
if (!hf_repo.empty()) {
return hf_repo;
@@ -1019,6 +1031,8 @@ struct llama_context_params common_context_params_to_llama(const common_params &
// apply --fit to mparams and cparams, no-op if params.fit_params is false
void common_fit_apply(common_params & params, struct llama_model_params & mparams, struct llama_context_params & cparams);
+struct llama_model * common_model_load(const std::vector<std::string> & paths, const struct llama_model_params & mparams);
+
// clear LoRA adapters from context, then apply new list of adapters
void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);
diff --git a/common/fit.cpp b/common/fit.cpp
index 10d57f956..7ab89cc09 100644
--- a/common/fit.cpp
+++ b/common/fit.cpp
@@ -56,7 +56,7 @@ private:
};
static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
- const char * path_model,
+ const std::vector<std::string> & paths_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
@@ -71,7 +71,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
mparams_copy.no_alloc = true;
mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
- llama_model * model = llama_model_load_from_file(path_model, mparams_copy);
+ llama_model * model = common_model_load(paths_model, mparams_copy);
if (model == nullptr) {
throw std::runtime_error("failed to load model");
}
@@ -167,7 +167,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
}
std::vector<common_memory_breakdown_row> common_fit_measure(
- const char * path_model,
+ const std::vector<std::string> & paths_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
uint32_t * n_ctx_train,
@@ -179,7 +179,7 @@ std::vector<common_memory_breakdown_row> common_fit_measure(
mparams_copy.no_alloc = true;
mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
- llama_model_ptr model(llama_model_load_from_file(path_model, mparams_copy));
+ llama_model_ptr model(common_model_load(paths_model, mparams_copy));
if (!model) {
throw std::runtime_error("failed to load model");
}
@@ -201,7 +201,7 @@ std::vector<common_memory_breakdown_row> common_fit_measure(
common_memory_reports common_measure_memory(common_params & params) {
const ggml_log_level log_level = params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR;
- const char * path_model = params.model.path.c_str();
+ const std::vector<std::string> paths_model = params.model.get_paths();
common_memory_reports res;
@@ -213,7 +213,7 @@ common_memory_reports common_measure_memory(common_params & params) {
auto mparams_fit = mparams;
auto cparams_fit = cparams;
common_fit_apply(params, mparams_fit, cparams_fit);
- res.info_max = common_fit_measure(path_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level);
+ res.info_max = common_fit_measure(paths_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level);
}
// min: the smallest context --fit may pick (--fit only changes n_ctx when it is 0)
@@ -231,14 +231,14 @@ common_memory_reports common_measure_memory(common_params & params) {
res.info_min = res.info_max;
} else {
cparams.n_ctx = n_ctx_min;
- res.info_min = common_fit_measure(path_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level);
+ res.info_min = common_fit_measure(paths_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level);
}
return res;
}
common_device_memory_data_vec common_get_device_memory_data(
- const char * path_model,
+ const std::vector<std::string> & paths_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
@@ -248,7 +248,7 @@ common_device_memory_data_vec common_get_device_memory_data(
uint32_t & hp_n_expert,
ggml_log_level log_level) {
std::vector<llama_device_memory_data> impl = common_get_device_memory_data_impl(
- path_model, mparams, cparams, devs, hp_ngl, hp_ngl_skip, hp_n_ctx_train, hp_n_expert, log_level);
+ paths_model, mparams, cparams, devs, hp_ngl, hp_ngl_skip, hp_n_ctx_train, hp_n_expert, log_level);
common_device_memory_data_vec ret(impl.size());
for (size_t i = 0; i < impl.size(); i++) {
@@ -262,7 +262,7 @@ common_device_memory_data_vec common_get_device_memory_data(
}
static void common_params_fit_impl(
- const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
+ const std::vector<std::string> & paths_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,
size_t * margins_s, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) {
if (mparams->split_mode == LLAMA_SPLIT_MODE_TENSOR) {
@@ -308,7 +308,7 @@ static void common_params_fit_impl(
dmds_t measured;
try {
measured = common_get_device_memory_data_impl(
- extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nsk_extra, nct_extra, nex_extra, log_level);
+ extra->paths_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nsk_extra, nct_extra, nex_extra, log_level);
} catch (const std::runtime_error & e) {
// the extra model is optional, fit the main model alone rather than giving up
LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what());
@@ -348,7 +348,7 @@ static void common_params_fit_impl(
// step 1: get data for default parameters and check whether any changes are necessary in the first place
LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
- dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
+ dmds_t dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
// saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min:
const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_streams, UINT32_MAX);
@@ -360,7 +360,7 @@ static void common_params_fit_impl(
if (n_streams > 1) {
LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n",
__func__, n_ctx_max, n_streams);
- dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
+ dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
}
}
add_extra_memory(dmds_full);
@@ -499,7 +499,7 @@ static void common_params_fit_impl(
int64_t sum_projected_used_min_ctx = 0;
cparams->n_ctx = n_ctx_min_total;
- dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
+ dmds_t dmds_min_ctx = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
add_extra_memory(dmds_min_ctx);
if (nd == 0) {
sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total();
@@ -695,7 +695,7 @@ static void common_params_fit_impl(
set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);
dmds_t dmd_nl = common_get_device_memory_data_impl(
- path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
+ paths_model, &mparams_copy, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
add_extra_memory(dmd_nl);
LOG_TRC("%s: memory for test allocation by device:\n", func_name);
@@ -724,7 +724,7 @@ static void common_params_fit_impl(
LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(
- path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
+ paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
add_extra_memory(dmds_cpu_moe);
for (size_t id = 0; id < nd; id++) {
@@ -978,7 +978,7 @@ static void common_params_fit_impl(
}
enum common_params_fit_status common_fit_params(
- const char * path_model,
+ const std::vector<std::string> & paths_model,
llama_model_params * mparams,
llama_context_params * cparams,
float * tensor_split,
@@ -990,7 +990,7 @@ enum common_params_fit_status common_fit_params(
const int64_t t0_us = llama_time_us();
common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS;
try {
- common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level);
+ common_params_fit_impl(paths_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level);
LOG_TRC("%s: successfully fit params to free device memory\n", __func__);
} catch (const common_params_fit_exception & e) {
LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what());
@@ -1200,7 +1200,7 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
}
void common_fit_print(
- const char * path_model,
+ const std::vector<std::string> & paths_model,
llama_model_params * mparams,
llama_context_params * cparams) {
std::vector<ggml_backend_dev_t> devs;
@@ -1209,7 +1209,7 @@ void common_fit_print(
uint32_t hp_nct = 0; // hparams.n_ctx_train
uint32_t hp_nex = 0; // hparams.n_expert
- auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
+ auto dmd = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
GGML_ASSERT(dmd.size() == devs.size() + 1);
for (size_t id = 0; id < devs.size(); id++) {
diff --git a/common/fit.h b/common/fit.h
index b2be5e4aa..431c41836 100644
--- a/common/fit.h
+++ b/common/fit.h
@@ -16,7 +16,7 @@ enum common_params_fit_status {
// - its context follows the context of the main model, so its memory is measured again whenever that context changes
// - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context
struct common_fit_extra_model {
- const char * path_model;
+ std::vector<std::string> paths_model;
llama_model_params * mparams;
llama_context_params * cparams;
bool shares_model;
@@ -28,19 +28,19 @@ struct common_fit_extra_model {
// - only parameters that have the same value as in llama_default_model_params are modified
// with the exception of the context size which is modified if and only if equal to 0
common_params_fit_status common_fit_params(
- const char * path_model,
+ const std::vector<std::string> & paths_model,
llama_model_params * mparams,
llama_context_params * cparams,
float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
size_t * margins, // margins of memory to leave per device in bytes
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
- const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none
+ const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none
ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
// print estimated memory to stdout
void common_fit_print(
- const char * path_model,
+ const std::vector<std::string> & paths_model,
llama_model_params * mparams,
llama_context_params * cparams);
@@ -75,7 +75,7 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const llam
// load model + context with no_alloc and return the memory breakdown, params are used as-is
// n_ctx_train and n_ctx are optional outputs; throws on failure
std::vector<common_memory_breakdown_row> common_fit_measure(
- const char * path_model,
+ const std::vector<std::string> & paths_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
uint32_t * n_ctx_train,
@@ -99,7 +99,7 @@ using common_device_memory_data_vec = std::vector<common_device_memory_data>;
// Load a model + context with no_alloc and return the per-device memory breakdown.
common_device_memory_data_vec common_get_device_memory_data(
- const char * path_model,
+ const std::vector<std::string> & paths_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
diff --git a/common/speculative.cpp b/common/speculative.cpp
index 6b5c9bc36..4006363dc 100644
--- a/common/speculative.cpp
+++ b/common/speculative.cpp
@@ -2858,7 +2858,7 @@ common_speculative_init_result::common_speculative_init_result(
model_path = params.speculative.draft.mparams.path;
LOG_INF("%s: loading draft model '%s'\n", __func__, model_path.c_str());
- llama_model * model_dft = llama_model_load_from_file(model_path.c_str(), mparams);
+ llama_model * model_dft = common_model_load(params.speculative.draft.mparams.get_paths(), mparams);
if (model_dft == NULL) {
LOG_ERR("%s: failed to load draft model, '%s'\n", __func__, model_path.c_str());
return;
diff --git a/conversion/base.py b/conversion/base.py
index f31297459..2cfdba30b 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -133,7 +133,8 @@ class ModelBase:
target_model_dir: Path | None = None,
fuse_gate_up_exps: bool = False,
fp8_as_q8: bool = False,
- fuse_qkv: bool = False):
+ fuse_qkv: bool = False,
+ ple_only: bool = False, no_ple: bool = False):
if type(self) is ModelBase or \
type(self) is TextModel or \
type(self) is MmprojModel:
@@ -142,6 +143,11 @@ class ModelBase:
if self.is_mistral_format and not _mistral_common_installed:
raise ImportError(_mistral_import_error_msg)
+ if ple_only and no_ple:
+ raise ValueError("ple_only and no_ple are mutually exclusive")
+ self.ple_only = ple_only
+ self.no_ple = no_ple
+ self._has_ple = False
self.dir_model = dir_model
self.ftype = ftype
self.fname_out = fname_out
@@ -204,7 +210,8 @@ class ModelBase:
# Configure GGUF Writer
self.gguf_writer = gguf.GGUFWriter(path=None, arch=gguf.MODEL_ARCH_NAMES[self.model_arch], endianess=self.endianess, use_temp_file=self.use_temp_file,
- split_max_tensors=split_max_tensors, split_max_size=split_max_size, dry_run=dry_run, small_first_shard=small_first_shard)
+ split_max_tensors=split_max_tensors, split_max_size=split_max_size, dry_run=dry_run, small_first_shard=small_first_shard,
+ tensor_filter=self._include_ple_tensor if ple_only or no_ple else None)
# Mistral specific
self.disable_mistral_community_chat_template = disable_mistral_community_chat_template
@@ -1382,6 +1389,11 @@ class ModelBase:
del experts, merged
+ def _include_ple_tensor(self, name: str) -> bool:
+ is_ple = name == gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD] + ".weight"
+ self._has_ple |= is_ple
+ return is_ple if self.ple_only else not is_ple
+
def prepare_tensors(self):
# detect NVFP4 quantization (ModelOpt and Compressed-tensors formats)
quantization_config = self.hparams.get("quantization_config") or {}
@@ -1513,6 +1525,8 @@ class ModelBase:
break
for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):
+ if (self.ple_only or self.no_ple) and not self._include_ple_tensor(new_name):
+ continue
n_dims = len(data_torch.shape)
data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims)
@@ -1645,6 +1659,8 @@ class ModelBase:
def write(self):
self.prepare_tensors()
+ if (self.ple_only or self.no_ple) and not self._has_ple:
+ raise ValueError("model has no per-layer embedding table to export separately")
self.prepare_metadata(vocab_only=False)
self.gguf_writer.write_header_to_file(path=self.fname_out)
self.gguf_writer.write_kv_data_to_file()
@@ -1837,7 +1853,8 @@ class TextModel(ModelBase):
fname_default: str = gguf.naming_convention(self.metadata.name, self.metadata.basename, self.metadata.finetune, self.metadata.version, size_label=None, output_type=None, model_type="vocab")
# Use the default filename
- self.fname_out = self.fname_out / f"{fname_default}.gguf"
+ prefix = "ple-" if self.ple_only else "no-ple-" if self.no_ple else ""
+ self.fname_out = self.fname_out / f"{prefix}{fname_default}.gguf"
else:
# Output path is a custom defined templated filename
# Note: `not is_dir()` is used because `.is_file()` will not detect
@@ -1846,8 +1863,9 @@ class TextModel(ModelBase):
# Process templated file name with the output ftype, useful with the "auto" ftype
self.fname_out = self.fname_out.parent / gguf.fill_templated_filename(self.fname_out.name, output_type)
- logger.info("Set model tokenizer")
- self.set_vocab()
+ if not self.ple_only:
+ logger.info("Set model tokenizer")
+ self.set_vocab()
def set_gguf_parameters(self):
self.gguf_writer.add_block_count(self.block_count)
diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index b863bccc5..8cee1f864 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -125,6 +125,15 @@ def parse_args() -> argparse.Namespace:
"--no-nextn", "--no-mtp", dest="no_mtp", action="store_true",
help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.",
)
+ ple_group = parser.add_mutually_exclusive_group()
+ ple_group.add_argument(
+ "--ple-only", action="store_true",
+ help="Export only the per-layer embedding table as a separate GGUF for use with --shard (alias: --ple).",
+ )
+ ple_group.add_argument(
+ "--no-ple", action="store_true",
+ help="Exclude the per-layer embedding table while retaining its metadata. Load the table separately with --shard (alias: --ple).",
+ )
parser.add_argument(
"--dspark", action="store_true",
help="Export only the DeepSeek-V4 or Mistral Large 4 DSpark draft tensors as a separate GGUF.",
@@ -173,6 +182,10 @@ def parse_args() -> argparse.Namespace:
args = parser.parse_args()
if not args.print_supported_models and args.model is None:
parser.error("the following arguments are required: model")
+ if (args.ple_only or args.no_ple) and (args.vocab_only or args.mmproj or args.mtp or args.dspark):
+ parser.error("--ple-only / --no-ple cannot be combined with --vocab-only, --mmproj, --mtp, or --dspark")
+ if args.ple_only and args.no_tensor_first_split:
+ parser.error("--ple-only cannot be combined with --no-tensor-first-split")
return args
@@ -308,6 +321,7 @@ def main() -> None:
fuse_gate_up_exps=args.fuse_gate_up_exps,
fp8_as_q8=args.fp8_as_q8,
fuse_qkv=args.fuse_qkv,
+ ple_only=args.ple_only, no_ple=args.no_ple,
)
if args.vocab_only:
diff --git a/docs/models.md b/docs/models.md
index eee595263..f466d76e5 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -24,3 +24,23 @@ The Hugging Face platform provides a variety of online tools for converting, qua
- Use the [GGUF-my-LoRA space](https://huggingface.co/spaces/ggml-org/gguf-my-lora) to convert LoRA adapters to GGUF format (more info: https://github.com/ggml-org/llama.cpp/discussions/10123)
- Use the [GGUF-editor space](https://huggingface.co/spaces/CISCai/gguf-editor) to edit GGUF meta data in the browser (more info: https://github.com/ggml-org/llama.cpp/discussions/9268)
- Use the [Inference Endpoints](https://ui.endpoints.huggingface.co/) to directly host `llama.cpp` in the cloud (more info: https://github.com/ggml-org/llama.cpp/discussions/9669)
+
+## Exporting per-layer embeddings separately
+
+For models with a per-layer embedding (PLE) table, export the backbone and table from the same checkpoint:
+
+```sh
+python3 convert_hf_to_gguf.py /path/to/model --no-ple --outfile model-no-ple.gguf
+python3 convert_hf_to_gguf.py /path/to/model --ple-only --outfile model-ple.gguf
+llama-cli -m model-no-ple.gguf --shard model-ple.gguf
+```
+
+`--no-ple` excludes only `per_layer_token_embd.weight` and keeps the PLE metadata and projection weights. `--ple-only` exports only that table, without a tokenizer. Each export supports `--outtype`, so the table and backbone can use different precisions. Default output names start with `no-ple-` and `ple-` respectively.
+
+These modes are mutually exclusive and require a model with a PLE table, such as Qwen3.8 Flash Next or Gemma 4. They cannot be combined with vocabulary, multimodal projector, MTP-only, or DSpark-only export. `--no-ple` can be combined with `--no-mtp`.
+
+`--ple` is an alias for `--shard`. Extra shards can be comma separated or supplied with repeated options, for example `-m backbone.gguf --shard /fast/ple.gguf,/other/token-embd.gguf`. Paths may be on different drives. Tensor names must be unique across all files, except for the PLE case below.
+
+A numbered backbone is auto-discovered and checked as usual; explicitly supplied paths outside that numbered set are appended as extra shards. Their split numbering is ignored, and the loader uses the actual combined tensor count. Paths already in the discovered set are loaded once. If the main filename does not follow the numbered naming scheme, an explicit backbone list must still match its declared split count and order.
+
+If an extra shard holds only `per_layer_token_embd.weight`, it replaces the PLE table of the model. If the model already has the table, the extra shard overrides it. A backbone split that holds only that tensor is skipped, and it can be absent from disk. This lets one PLE file be shared across quants and releases, for example `-m model-Q4_K_M-00001-of-00003.gguf --shard /fast/ple-Q8_0.gguf`.
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 21b354ca4..364eeaa75 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -11,7 +11,7 @@ from enum import Enum, auto
from math import prod
from pathlib import Path
from io import BufferedWriter
-from typing import IO, Any, Sequence, Mapping
+from typing import IO, Any, Callable, Sequence, Mapping
from string import ascii_letters, digits
import numpy as np
@@ -86,7 +86,8 @@ class GGUFWriter:
def __init__(
self, path: os.PathLike[str] | str | None, arch: str, use_temp_file: bool = False, endianess: GGUFEndian = GGUFEndian.LITTLE,
- split_max_tensors: int = 0, split_max_size: int = 0, dry_run: bool = False, small_first_shard: bool = False
+ split_max_tensors: int = 0, split_max_size: int = 0, dry_run: bool = False, small_first_shard: bool = False,
+ tensor_filter: Callable[[str], bool] | None = None,
):
self.fout = None
self.path = Path(path) if path else None
@@ -101,6 +102,7 @@ class GGUFWriter:
self.split_max_size = split_max_size
self.dry_run = dry_run
self.small_first_shard = small_first_shard
+ self.tensor_filter = tensor_filter
logger.info("gguf: This GGUF file is for {0} Endian only".format(
"Big" if self.endianess == GGUFEndian.BIG else "Little",
))
@@ -381,6 +383,9 @@ class GGUFWriter:
self, name: str, tensor: np.ndarray[Any, Any], raw_shape: Sequence[int] | None = None,
raw_dtype: GGMLQuantizationType | None = None, tensor_endianess: GGUFEndian | None = None
) -> None:
+ if self.tensor_filter is not None and not self.tensor_filter(name):
+ return
+
# if tensor endianness is not passed, assume it's native to system
if tensor_endianess is None:
tensor_endianess = GGUFEndian.BIG if sys.byteorder == 'big' else GGUFEndian.LITTLE
diff --git a/gguf-py/tests/test_gguf_writer.py b/gguf-py/tests/test_gguf_writer.py
new file mode 100644
index 000000000..606b2c5f9
--- /dev/null
+++ b/gguf-py/tests/test_gguf_writer.py
@@ -0,0 +1,23 @@
+import numpy as np
+import pytest
+
+from gguf import GGUFReader, GGUFWriter
+
+
+@pytest.mark.parametrize("use_temp_file", [False, True])
+def test_tensor_filter_preserves_data_offsets(tmp_path, use_temp_file):
+ path = tmp_path / "filtered.gguf"
+ data = np.arange(32, dtype=np.float32).reshape(1, 32)
+ writer = GGUFWriter(path, "qwen4exp", use_temp_file=use_temp_file,
+ tensor_filter=lambda name: name != "excluded")
+ writer.add_tensor("first", data)
+ writer.add_tensor("excluded", data + 100)
+ writer.add_tensor("last", data + 200)
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+ writer.write_tensors_to_file()
+ writer.close()
+ tensors = GGUFReader(path).tensors
+ assert [t.name for t in tensors] == ["first", "last"]
+ np.testing.assert_array_equal(tensors[0].data, data)
+ np.testing.assert_array_equal(tensors[1].data, data + 200)
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
index 485483ba1..2cbee61d4 100644
--- a/src/llama-model-loader.cpp
+++ b/src/llama-model-loader.cpp
@@ -597,85 +597,167 @@ llama_model_loader::llama_model_loader(
n_bytes += ggml_nbytes(cur);
weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur));
}
+ auto paths = splits;
uint16_t n_split = 0;
+ size_t n_backbone = 1;
+ const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO);
get_key(llm_kv(LLM_KV_SPLIT_COUNT), n_split, false);
- // Load additional GGML contexts
if (n_split > 1) {
- // make sure the main file is loaded first
uint16_t idx = 0;
- const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO);
get_key(kv_split_no, idx);
if (idx != 0) {
throw std::runtime_error(format("illegal split file idx: %d (file: %s), model must be loaded with the first split", idx, fname.c_str()));
}
- // generate list of splits if needed
- if (splits.empty()) {
- splits = llama_get_list_splits(fname, idx, n_split);
+ std::vector<char> prefix(llama_path_max(), 0);
+ if (paths.empty() || llama_split_prefix(prefix.data(), prefix.size(), fname.c_str(), idx, n_split)) {
+ auto backbone = llama_get_list_splits(fname, idx, n_split);
+ auto file_name = [](const std::string & path) {
+ return path.substr(path.find_last_of("/\\") + 1);
+ };
+ std::vector<std::string> extras;
+ for (const auto & path : paths) {
+ auto it = std::find_if(backbone.begin(), backbone.end(), [&](const std::string & b) {
+ return file_name(b) == file_name(path);
+ });
+ if (it != backbone.end()) {
+ *it = path; // given path wins, splits can be in different dirs
+ } else {
+ extras.push_back(path);
+ }
+ }
+ paths = std::move(backbone);
+ paths.insert(paths.end(), extras.begin(), extras.end());
+ } else if (paths.size() != n_split) {
+ throw std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", paths.size(), n_split));
}
+ n_backbone = n_split;
+ } else if (paths.empty()) {
+ paths.push_back(fname);
+ }
- // in case user give a custom list of splits, check if it matches the expected number
- if (n_split != (uint16_t)splits.size()) {
- throw std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", splits.size(), n_split));
- }
+ if (paths.size() > UINT16_MAX) {
+ throw std::runtime_error("too many model files");
+ }
+ const bool has_extras = paths.size() > n_backbone;
- if (trace > 0) {
- LLAMA_LOG_INFO("%s: loading additional %d GGUFs\n", __func__, n_split);
+ // special case for PLE-only GGUFs: no need to follow the exact split numbering (allow flexibility)
+ // ref: https://github.com/ggml-org/llama.cpp/pull/31185
+ auto is_ple_only = [](const gguf_context * ctx) {
+ return gguf_get_n_tensors(ctx) == 1 && std::strcmp(gguf_get_tensor_name(ctx, 0), "per_layer_token_embd.weight") == 0;
+ };
+ bool skip_shard_order_validation = false;
+ for (size_t idx = n_backbone; idx < paths.size(); idx++) {
+ struct gguf_init_params probe_params = {
+ /*.no_alloc = */ true,
+ /*.ctx = */ nullptr,
+ };
+ gguf_context_ptr probe { gguf_init_from_file(paths[idx].c_str(), probe_params) };
+ skip_shard_order_validation |= probe && is_ple_only(probe.get());
+ }
+
+ size_t n_skipped = 0;
+ size_t n_backbone_files = 1;
+ int n_backbone_tensors = gguf_get_n_tensors(metadata);
+ for (size_t idx = 1; idx < paths.size(); idx++) {
+ const char * fname_split = paths[idx].c_str();
+ if (skip_shard_order_validation && idx < n_backbone) {
+ FILE * f = ggml_fopen(fname_split, "rb");
+ if (f == nullptr) {
+ LLAMA_LOG_INFO("%s: GGUF split %s not found, assuming it only holds the per-layer embedding table\n", __func__, fname_split);
+ n_skipped++;
+ continue;
+ }
+ fclose(f);
+ }
+ ctx = nullptr; // still owned by contexts, not set by gguf on failure
+ struct gguf_init_params split_params = {
+ /*.no_alloc = */ true,
+ /*.ctx = */ &ctx,
+ };
+ gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) };
+ ggml_context_ptr ctx_ptr(ctx);
+ if (!ctx_gguf) {
+ throw std::runtime_error(format("%s: failed to load GGUF split from %s", __func__, fname_split));
}
- // load other splits
- for (idx = 1; idx < n_split; idx++) {
- const char * fname_split = splits[idx].c_str();
-
- struct gguf_init_params split_params = {
- /*.no_alloc = */ true,
- /*.ctx = */ &ctx,
- };
- gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) };
- if (!ctx_gguf) {
- throw std::runtime_error(format("%s: failed to load GGUF split from %s", __func__, fname_split));
+ bool allow_override_tensor = false;
+ if (idx < n_backbone) {
+ const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str());
+ if (kid < 0) {
+ throw std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split));
}
-
- // check idx
- {
- const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str());
- if (kid < 0) {
- throw std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split));
- }
- int idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid);
- if (idx_gguf != idx) {
- throw std::runtime_error(format("invalid split file idx: %d (file: %s), expected %d", idx_gguf, fname_split, idx));
- }
+ if (gguf_get_kv_type(ctx_gguf.get(), kid) != GGUF_TYPE_UINT16) {
+ throw std::runtime_error(format("invalid type for %s in GGUF split %s", kv_split_no.c_str(), fname_split));
+ }
+ const uint16_t idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid);
+ if (idx_gguf != idx) {
+ throw std::runtime_error(format("invalid split file idx: %d (file: %s), expected %zu", idx_gguf, fname_split, idx));
}
+ if (skip_shard_order_validation && is_ple_only(ctx_gguf.get())) {
+ LLAMA_LOG_INFO("%s: skipping GGUF split %s, per-layer embedding table given separately\n", __func__, fname_split);
+ n_skipped++;
+ continue;
+ }
+ n_backbone_tensors += gguf_get_n_tensors(ctx_gguf.get());
+ } else {
+ // an external PLE table replaces the one in the model, if any
+ allow_override_tensor = is_ple_only(ctx_gguf.get());
+ const bool numbered = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str()) >= 0 ||
+ gguf_find_key(ctx_gguf.get(), llm_kv(LLM_KV_SPLIT_COUNT).c_str()) >= 0 ||
+ gguf_find_key(ctx_gguf.get(), llm_kv(LLM_KV_SPLIT_TENSORS_COUNT).c_str()) >= 0;
+ LLAMA_LOG_INFO("%s: appended extra GGUF shard %s%s\n", __func__, fname_split,
+ numbered ? " (declared split numbering ignored)" : "");
+ }
- files.emplace_back(new llama_file(fname_split, "rb", use_direct_io));
- contexts.emplace_back(ctx);
+ files.emplace_back(new llama_file(fname_split, "rb", use_direct_io));
+ contexts.emplace_back(std::move(ctx_ptr));
+ if (idx < n_backbone) {
+ n_backbone_files = files.size();
+ }
- // Save tensors data offset info of the shard.
- for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
- std::string tensor_name = std::string(cur->name);
- // make sure there is no duplicated tensor names
- if (weights_map.find(tensor_name) != weights_map.end()) {
+ // skipped splits are not in files, so idx != file index
+ const uint16_t file_idx = files.size() - 1;
+ for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
+ const std::string tensor_name = cur->name;
+ auto it = weights_map.find(tensor_name);
+ if (it != weights_map.end()) {
+ // only override tensors of the model, not of another extra shard
+ if (!allow_override_tensor || it->second.idx >= n_backbone_files) {
throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));
}
- n_elements += ggml_nelements(cur);
- n_bytes += ggml_nbytes(cur);
- weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), idx, ctx_gguf.get(), cur));
+ LLAMA_LOG_INFO("%s: tensor '%s' overridden by %s\n", __func__, tensor_name.c_str(), fname_split);
+ n_elements -= ggml_nelements(it->second.tensor);
+ n_bytes -= ggml_nbytes(it->second.tensor);
+ weights_map.erase(it);
}
+ n_elements += ggml_nelements(cur);
+ n_bytes += ggml_nbytes(cur);
+ weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), file_idx, ctx_gguf.get(), cur));
}
+ }
- get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), n_tensors);
-
- // sanity check
- {
- const int n_tensors_loaded = (int) weights_map.size();
- if (n_tensors != n_tensors_loaded) {
- throw std::runtime_error(format("corrupted model: %d tensors expected but %d found", n_tensors, n_tensors_loaded));
+ if (n_skipped > 0) {
+ // the external table can only replace one tensor of the numbered set
+ int declared_tensors = 0;
+ get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), declared_tensors);
+ if (n_skipped != 1 || n_backbone_tensors + 1 != declared_tensors) {
+ throw std::runtime_error(format("incomplete model: %zu split(s) skipped, %d of %d tensors found in the split files", n_skipped, n_backbone_tensors, declared_tensors));
+ }
+ }
+ if (n_split > 1 || has_extras) {
+ int declared_tensors = gguf_get_n_tensors(metadata);
+ get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), declared_tensors, n_split > 1 && !has_extras);
+ if (declared_tensors != (int) weights_map.size()) {
+ if (n_split > 1 && !has_extras) {
+ throw std::runtime_error(format("corrupted model: %d tensors expected but %zu found", declared_tensors, weights_map.size()));
}
+ LLAMA_LOG_INFO("%s: using %zu registered tensors instead of %d declared tensors\n", __func__, weights_map.size(), declared_tensors);
}
-
- LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split - 1);
+ }
+ if (files.size() > 1) {
+ LLAMA_LOG_INFO("%s: additional %zu GGUFs metadata loaded.\n", __func__, files.size() - 1);
}
} else if (file != nullptr) {
struct ggml_context * ctx = NULL;
diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp
index 543da2aed..537c33394 100644
--- a/tests/test-arg-parser.cpp
+++ b/tests/test-arg-parser.cpp
@@ -245,6 +245,26 @@ static void test(void) {
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
assert(params.model.path == "model_file.gguf");
+ {
+ auto parse_shards = [&list_str_to_char](std::vector<std::string> args) {
+ common_params shard_params;
+ args.insert(args.begin(), {"binary_name", "-m", "model_file.gguf"});
+ assert(common_params_parse(args.size(), list_str_to_char(args).data(), shard_params, LLAMA_EXAMPLE_COMMON));
+ auto paths = shard_params.model.get_paths();
+ assert(paths.front() == "model_file.gguf");
+ return std::vector<std::string>(paths.begin() + 1, paths.end());
+ };
+ assert(parse_shards({"--shard", "a.gguf"}) == std::vector<std::string>({"a.gguf"}));
+ const std::vector<std::string> two = {"a.gguf", "b.gguf"};
+ assert(parse_shards({"--shard", "a.gguf", "--shard", "b.gguf"}) == two);
+ assert(parse_shards({"--shard", "a.gguf,b.gguf"}) == two);
+ const std::vector<std::string> three = {"a.gguf", "b.gguf", "c.gguf"};
+ assert(parse_shards({"--shard", "a.gguf,b.gguf", "--shard", "c.gguf"}) == three);
+ assert(parse_shards({"--shard", "a.gguf", "--shard", "b.gguf", "--shard", "c.gguf"}) == three);
+ assert(parse_shards({"--ple", "p.gguf"}) == parse_shards({"--shard", "p.gguf"}));
+ assert(parse_shards({"--shard", "model_file.gguf,a.gguf,b.gguf,a.gguf"}) == two);
+ }
+
argv = {"binary_name", "-t", "1234"};
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
assert(params.cpuparams.n_threads == 1234);
diff --git a/tools/fit-params/fit-params.cpp b/tools/fit-params/fit-params.cpp
index 3e78c8929..3dc46069c 100644
--- a/tools/fit-params/fit-params.cpp
+++ b/tools/fit-params/fit-params.cpp
@@ -31,7 +31,7 @@ int llama_fit_params(int argc, char ** argv) {
auto cparams = common_context_params_to_llama(params);
if (!params.fit_params_print) {
- const common_params_fit_status status = common_fit_params(params.model.path.c_str(), &mparams, &cparams,
+ const common_params_fit_status status = common_fit_params(params.model.get_paths(), &mparams, &cparams,
params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx,
nullptr,
params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
@@ -71,7 +71,7 @@ int llama_fit_params(int argc, char ** argv) {
LOG_INF("%s: printing estimated memory in MiB to stdout (device, model, context, compute) ...\n", __func__);
common_log_flush(common_log_main());
- common_fit_print(params.model.path.c_str(), &mparams, &cparams);
+ common_fit_print(params.model.get_paths(), &mparams, &cparams);
}
return 0;
diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp
index 96ae4870f..bd87644c3 100644
--- a/tools/llama-bench/llama-bench.cpp
+++ b/tools/llama-bench/llama-bench.cpp
@@ -2352,7 +2352,7 @@ int llama_bench(int argc, char ** argv) {
uint32_t n_ctx_needed = std::max<uint32_t>(inst.n_prompt + inst.n_gen + inst.n_depth, inst.fit_min_ctx);
cparams.n_ctx = std::max(cparams.n_ctx, n_ctx_needed);
- common_fit_params(inst.model.c_str(), &mparams, &cparams,
+ common_fit_params({ inst.model }, &mparams, &cparams,
fit_tensor_split.data(),
fit_overrides.data(),
margins.data(),
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
index fa0f09e71..dc277ead2 100644
--- a/tools/server/server-models.cpp
+++ b/tools/server/server-models.cpp
@@ -279,6 +279,7 @@ static void unset_reserved_args(common_preset & preset, bool unset_model_args) {
preset.unset_option("LLAMA_ARG_CONNECT_CODE");
if (unset_model_args) {
preset.unset_option("LLAMA_ARG_MODEL");
+ preset.unset_option("LLAMA_ARG_SHARD");
preset.unset_option("LLAMA_ARG_MMPROJ");
preset.unset_option("LLAMA_ARG_ALIAS");
preset.unset_option("LLAMA_ARG_HF_REPO");