Commit 7144124a5 for llama.cpp
commit 7144124a5b60551fc2b6af290d0e433c01c95115
Author: Daniel Bevenius <daniel.bevenius@gmail.com>
Date: Wed Nov 18 11:33:12 2026 +0100
vocab : extract bpe_rank loading into function (#30962)
* vocab : extract bpe_rank loading into function
This commit extracts the BPE rank loading logic into a separate function
named `load_bpe_ranks`.
The motivation for this is that the came code is currently used by both
gpt2/hybriddna/whitespace tokenizer models and gemma4 and this avoids
the duplication of code to improve readability and maintainability.
* change signature of load_bpe_ranks
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 2bb183d21..3aecacb51 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -2241,9 +2241,41 @@ struct llama_vocab::impl {
void print_info() const;
private:
+ bool load_bpe_ranks(const gguf_context * ctx, const LLM_KV & kv);
+
const llama_vocab & vocab;
};
+bool llama_vocab::impl::load_bpe_ranks(const gguf_context * ctx, const LLM_KV & kv) {
+ const int merges_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_MERGES).c_str());
+ if (merges_keyidx == -1) {
+ return false;
+ }
+
+ if (gguf_get_kv_type(ctx, merges_keyidx) != GGUF_TYPE_ARRAY ||
+ gguf_get_arr_type(ctx, merges_keyidx) != GGUF_TYPE_STRING) {
+ throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_MERGES).c_str()));
+ }
+
+ const int n_merges = gguf_get_arr_n(ctx, merges_keyidx);
+ bpe_ranks.reserve(n_merges);
+ for (int i = 0; i < n_merges; i++) {
+ const char * word = gguf_get_arr_str(ctx, merges_keyidx, i);
+ std::string first;
+ std::string second;
+
+ const char * sep = *word ? strchr(word + 1, ' ') : nullptr;
+
+ if (sep) {
+ first.assign(word, sep - word);
+ second.assign(sep + 1);
+ }
+
+ bpe_ranks.emplace(std::make_pair(std::move(first), std::move(second)), i);
+ }
+ return true;
+}
+
void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
struct gguf_context * ctx = ml.metadata;
@@ -2301,40 +2333,16 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} else if (tokenizer_model == "gpt2" || tokenizer_model == "hybriddna" || tokenizer_model == "whitespace") {
type = LLAMA_VOCAB_TYPE_BPE;
- // read bpe merges and populate bpe ranks
- const int merges_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_MERGES).c_str());
// Kimi-K2 uses custom tokenization without traditional BPE merges
const bool is_kimi_k2 = (tokenizer_pre == "kimi-k2");
- if (merges_keyidx == -1) {
+ // read bpe merges and populate bpe ranks
+ if (!load_bpe_ranks(ctx, kv)) {
if (!is_kimi_k2) {
throw std::runtime_error("cannot find tokenizer merges in model file\n");
}
// Kimi-K2 doesn't need merges, skip
LLAMA_LOG_INFO("%s: Kimi-K2 tokenizer detected, skipping BPE merges\n", __func__);
- } else {
- if (gguf_get_kv_type(ctx, merges_keyidx) != GGUF_TYPE_ARRAY ||
- gguf_get_arr_type(ctx, merges_keyidx) != GGUF_TYPE_STRING) {
- throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_MERGES).c_str()));
- }
- const int n_merges = gguf_get_arr_n(ctx, merges_keyidx);
- bpe_ranks.reserve(n_merges);
- for (int i = 0; i < n_merges; i++) {
- const char * word = gguf_get_arr_str(ctx, merges_keyidx, i);
- //GGML_ASSERT(unicode_cpts_from_utf8(word).size() > 0);
-
- std::string first;
- std::string second;
-
- const char * sep = *word ? strchr(word + 1, ' ') : nullptr;
-
- if (sep) {
- first.assign(word, sep - word);
- second.assign(sep + 1);
- }
-
- bpe_ranks.emplace(std::make_pair(std::move(first), std::move(second)), i);
- }
}
// default special tokens
@@ -2420,33 +2428,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
type = LLAMA_VOCAB_TYPE_BPE;
// read bpe merges and populate bpe ranks
- const int merges_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_MERGES).c_str());
- if (merges_keyidx == -1) {
+ if (!load_bpe_ranks(ctx, kv)) {
throw std::runtime_error("cannot find tokenizer merges in model file\n");
}
- {
- if (gguf_get_kv_type(ctx, merges_keyidx) != GGUF_TYPE_ARRAY ||
- gguf_get_arr_type(ctx, merges_keyidx) != GGUF_TYPE_STRING) {
- throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_MERGES).c_str()));
- }
- const int n_merges = gguf_get_arr_n(ctx, merges_keyidx);
- bpe_ranks.reserve(n_merges);
- for (int i = 0; i < n_merges; i++) {
- const char * word = gguf_get_arr_str(ctx, merges_keyidx, i);
-
- std::string first;
- std::string second;
-
- const char * sep = *word ? strchr(word + 1, ' ') : nullptr;
-
- if (sep) {
- first.assign(word, sep - word);
- second.assign(sep + 1);
- }
-
- bpe_ranks.emplace(std::make_pair(std::move(first), std::move(second)), i);
- }
- }
// default special tokens (to be read from GGUF)
special_bos_id = LLAMA_TOKEN_NULL;