Commit d88040c70 for llama.cpp
commit d88040c704c8855c497d46c20675a3d492f70f94
Author: tkran2 <157059462+tkran2@users.noreply.github.com>
Date: Thu Oct 22 02:58:08 2026 -0500
vocab : add clean_spaces GGUF key (#30940)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 51a16ef52..4482f7f33 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -371,6 +371,7 @@ class Keys:
DNA_AUTO_TAGS = "tokenizer.ggml.dna_auto_tags"
ADD_PREFIX = "tokenizer.ggml.add_space_prefix"
REMOVE_EXTRA_WS = "tokenizer.ggml.remove_extra_whitespaces"
+ CLEAN_SPACES = "tokenizer.ggml.clean_spaces"
PRECOMPILED_CHARSMAP = "tokenizer.ggml.precompiled_charsmap"
SUPPRESS_TOKENS = "tokenizer.ggml.suppress_tokens"
HF_JSON = "tokenizer.huggingface.json"
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index d00392211..2faaefa59 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1326,6 +1326,9 @@ class GGUFWriter:
def add_remove_extra_whitespaces(self, value: bool) -> None:
self.add_bool(Keys.Tokenizer.REMOVE_EXTRA_WS, value)
+ def add_clean_spaces(self, value: bool) -> None:
+ self.add_bool(Keys.Tokenizer.CLEAN_SPACES, value)
+
def add_precompiled_charsmap(self, charsmap: bytes) -> None:
self.add_array(Keys.Tokenizer.PRECOMPILED_CHARSMAP, charsmap)
diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py
index 78e90eddc..6383178ea 100644
--- a/gguf-py/gguf/vocab.py
+++ b/gguf-py/gguf/vocab.py
@@ -55,6 +55,7 @@ class SpecialVocab:
normalizer_nfc: bool | None
normalizer_lowercase: bool | None
normalizer_strip_accents: bool | None
+ clean_spaces: bool | None
def __init__(
self, path: str | os.PathLike[str], load_merges: bool = False,
@@ -70,6 +71,7 @@ class SpecialVocab:
self.normalizer_nfc = None
self.normalizer_lowercase = None
self.normalizer_strip_accents = None
+ self.clean_spaces = None
if special_token_types is not None:
self.special_token_types = special_token_types
else:
@@ -120,6 +122,10 @@ class SpecialVocab:
if not quiet:
logger.info(f'Setting normalizer_strip_accents to {self.normalizer_strip_accents}')
gw.add_normalizer_strip_accents(self.normalizer_strip_accents)
+ if self.clean_spaces is not None:
+ if not quiet:
+ logger.info(f'Setting clean_spaces to {self.clean_spaces}')
+ gw.add_clean_spaces(self.clean_spaces)
def _load(self, path: Path) -> None:
self._try_load_from_tokenizer_json(path)
@@ -345,6 +351,10 @@ class SpecialVocab:
self.chat_template = chat_template
else:
logger.warning(f'Bad type for chat_template field in {tokenizer_config_file!r} - ignoring')
+ # decode clean-up of spaces before punctuation (transformers clean_up_tokenization_spaces)
+ clean_spaces = tokenizer_config.get('clean_up_tokenization_spaces')
+ if isinstance(clean_spaces, bool):
+ self.clean_spaces = clean_spaces
for typ in self.special_token_types:
add_entry = tokenizer_config.get(f'add_{typ}_token')
if isinstance(add_entry, bool):
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 62b012c5c..d79b18dec 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -437,6 +437,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_TOKENIZER_DNA_AUTO_TAGS, "tokenizer.ggml.dna_auto_tags" },
{ LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },
{ LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },
+ { LLM_KV_TOKENIZER_CLEAN_SPACES, "tokenizer.ggml.clean_spaces" },
{ LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },
{ LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },
{ LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 6713db6b5..12d8fe392 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -400,6 +400,7 @@ enum llm_kv {
LLM_KV_TOKENIZER_DNA_AUTO_TAGS,
LLM_KV_TOKENIZER_ADD_PREFIX,
LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,
+ LLM_KV_TOKENIZER_CLEAN_SPACES,
LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,
LLM_KV_TOKENIZER_HF_JSON,
LLM_KV_TOKENIZER_RWKV,
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index d620fcd9c..2135b82b1 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -424,6 +424,7 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_TOKENIZER_ADD_SEP, vocab.get_add_sep());
add_kv(LLM_KV_TOKENIZER_ADD_PREFIX, vocab.get_add_space_prefix());
add_kv(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, vocab.get_remove_extra_whitespaces());
+ add_kv(LLM_KV_TOKENIZER_CLEAN_SPACES, vocab.get_clean_spaces());
add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap());
// add_kv(LLM_KV_TOKENIZER_HF_JSON, ???);
// add_kv(LLM_KV_TOKENIZER_RWKV, ???);
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index d25c4ceec..80a5fcdff 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -2432,6 +2432,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} else if (tokenizer_pre == "minicpm5") {
pre_type = LLAMA_VOCAB_PRE_TYPE_MINICPM5;
ignore_merges = true;
+ clean_spaces = false;
} else if (
tokenizer_pre == "llama3" ||
tokenizer_pre == "llama-v3" ||
@@ -2439,11 +2440,16 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "falcon3" ||
tokenizer_pre == "falcon-h1" ||
tokenizer_pre == "midm-2.0" ||
- tokenizer_pre == "lfm2" ||
tokenizer_pre == "jina-v5-nano") {
pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
ignore_merges = true;
add_bos = true;
+ } else if (
+ tokenizer_pre == "lfm2") {
+ pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
+ ignore_merges = true;
+ add_bos = true;
+ clean_spaces = false;
} else if (
tokenizer_pre == "deepseek-llm") {
pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM;
@@ -2494,6 +2500,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "granite-embed-multi-311m") {
pre_type = LLAMA_VOCAB_PRE_TYPE_GEMMA4;
escape_whitespaces = true;
+ clean_spaces = false;
} else if (
tokenizer_pre == "mmbert") {
pre_type = LLAMA_VOCAB_PRE_TYPE_MMBERT;
@@ -2540,6 +2547,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} else if (
tokenizer_pre == "dbrx") {
pre_type = LLAMA_VOCAB_PRE_TYPE_DBRX;
+ clean_spaces = false;
} else if (
tokenizer_pre == "smaug-bpe") {
pre_type = LLAMA_VOCAB_PRE_TYPE_SMAUG;
@@ -2551,12 +2559,14 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "chatglm-bpe") {
pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4;
special_bos_id = LLAMA_TOKEN_NULL;
+ clean_spaces = false;
} else if (
tokenizer_pre == "glm4" ||
tokenizer_pre == "glm5") {
pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4;
special_bos_id = LLAMA_TOKEN_NULL;
ignore_merges = true;
+ clean_spaces = false;
} else if (
tokenizer_pre == "viking") {
pre_type = LLAMA_VOCAB_PRE_TYPE_VIKING;
@@ -2590,6 +2600,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} else if (
tokenizer_pre == "exaone4") {
pre_type = LLAMA_VOCAB_PRE_TYPE_GPT2;
+ clean_spaces = false;
} else if (
tokenizer_pre == "exaone-moe") {
pre_type = LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE;
@@ -2690,6 +2701,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} else if (
tokenizer_pre == "mellum2") {
pre_type = LLAMA_VOCAB_PRE_TYPE_MELLUM2;
+ clean_spaces = false;
} else if (
tokenizer_pre == "k2-horizon") {
pre_type = LLAMA_VOCAB_PRE_TYPE_K2_HORIZON;
@@ -2726,6 +2738,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
ml.get_key(LLM_KV_TOKENIZER_ADD_PREFIX, add_space_prefix, false);
ml.get_key(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, remove_extra_whitespaces, false);
+ ml.get_key(LLM_KV_TOKENIZER_CLEAN_SPACES, clean_spaces, false);
}
const int token_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_LIST).c_str());
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 24e6ba609..815cc15a3 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -142,7 +142,7 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-command-r ARGS ${PROJE
llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-coder ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-coder.gguf)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-llm ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-llm.gguf)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-falcon ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-falcon.gguf)
-llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf)
+llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf --check-decode)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-gpt-2 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gpt-2.gguf)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-bpe ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-bpe.gguf)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-spm ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-spm.gguf)
diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp
index 37f8312c4..5253a46a6 100644
--- a/tests/test-tokenizer-0.cpp
+++ b/tests/test-tokenizer-0.cpp
@@ -125,7 +125,7 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string &
int main(int argc, char **argv) {
if (argc < 2) {
- fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]);
+ fprintf(stderr, "Usage: %s vocab-file [text-file] [--check-decode]\n", argv[0]);
return 1;
}
@@ -135,8 +135,13 @@ int main(int argc, char **argv) {
const std::string fname_out = fname + ".out";
std::string fname_text;
- if (argc > 2) {
- fname_text = argv[2];
+ bool check_decode = false; // also check that the expected tokens detokenize to the input text
+ for (int i = 2; i < argc; i++) {
+ if (std::string(argv[i]) == "--check-decode") {
+ check_decode = true;
+ } else {
+ fname_text = argv[i];
+ }
}
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
@@ -244,6 +249,14 @@ int main(int argc, char **argv) {
success = false;
}
+
+ if (check_decode) {
+ const std::string dec = common_detokenize(ctx, test_kv.second);
+ if (dec != test_kv.first) {
+ fprintf(stderr, "%s : failed decode: '%s' detokenized to '%s'\n", __func__, test_kv.first.c_str(), dec.c_str());
+ success = false;
+ }
+ }
}
});
}