Commit e197cef45 for llama.cpp
commit e197cef4554d466e0becac20e453a355daba8e5d
Author: tkran2 <157059462+tkran2@users.noreply.github.com>
Date: Thu Oct 22 02:06:47 2026 -0500
vocab : use Tekken pre-tokenizer for pixtral rather than the Llama 3 one (#30960)
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 05c0f96a2..d25c4ceec 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -2438,7 +2438,6 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "llama-bpe"||
tokenizer_pre == "falcon3" ||
tokenizer_pre == "falcon-h1" ||
- tokenizer_pre == "pixtral" ||
tokenizer_pre == "midm-2.0" ||
tokenizer_pre == "lfm2" ||
tokenizer_pre == "jina-v5-nano") {
@@ -2566,7 +2565,8 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "jais") {
pre_type = LLAMA_VOCAB_PRE_TYPE_JAIS;
} else if (
- tokenizer_pre == "tekken") {
+ tokenizer_pre == "tekken" ||
+ tokenizer_pre == "pixtral") {
pre_type = LLAMA_VOCAB_PRE_TYPE_TEKKEN;
clean_spaces = false;
ignore_merges = true;