Commit 0693dbfb9 for llama.cpp
commit 0693dbfb915f33233818373b5f6969396658d02c
Author: martin-cimmino <159526529+martin-cimmino@users.noreply.github.com>
Date: Thu Oct 22 13:34:47 2026 +0200
nemotron : support non-standard GQA, BPE tokenizer and domyn-small pre-type (#23343)
* nemotron : support non-standard GQA, BPE tokenizer and domyn-small pre-type
* gguf-py : write Metaspace prepend_scheme as GGUF metadata
* vocab : parse metaspace prepend_scheme into enum, rename key
* vocab : use metaspace prepend_scheme in detokenize, add TODOs for other tokenizers
* vocab : use constant for metaspace escaped space
---------
Co-authored-by: Martin Cimmino <martin.cimmino@domyn.com>
diff --git a/conversion/base.py b/conversion/base.py
index f3bbe7b0f..d8420eb32 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -1837,6 +1837,9 @@ class TextModel(ModelBase):
if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":
# ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0
res = "minerva-7b"
+ if chkhsh == "6e4d72c6e5260b90606e26a0931fa15e4b42ec6e97719d12919f2b1f93478262":
+ # ref: https://huggingface.co/domyn/Domyn-Small-v1.0
+ res = "domyn-small"
if chkhsh == "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664":
# ref: https://huggingface.co/tencent/Hunyuan-A13B-Instruct
res = "hunyuan"
diff --git a/conversion/nemotron.py b/conversion/nemotron.py
index c90048194..d76a45d82 100644
--- a/conversion/nemotron.py
+++ b/conversion/nemotron.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import re
from typing import Any, Callable, Iterable, Sequence, TYPE_CHECKING
import numpy as np
@@ -367,9 +368,21 @@ class NemotronModel(TextModel):
model_arch = gguf.MODEL_ARCH.NEMOTRON
def set_vocab(self):
- self._set_vocab_sentencepiece()
- self.gguf_writer.add_pad_token_id(0)
- self.gguf_writer.add_unk_token_id(1)
+ if (self.dir_model / "tokenizer.model").exists():
+ self._set_vocab_sentencepiece()
+ self.gguf_writer.add_pad_token_id(0)
+ self.gguf_writer.add_unk_token_id(1)
+ else:
+ # SPM-style BPE (Metaspace + byte fallback): mark <0xXX> tokens as BYTE so they decode to raw bytes
+ tokens, toktypes, tokpre = self.get_vocab_base()
+ toktypes = [gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-F]{2}>", tok) else typ for tok, typ in zip(tokens, toktypes)]
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre(tokpre)
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+
+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+ special_vocab.add_to_gguf(self.gguf_writer)
def set_gguf_parameters(self):
super().set_gguf_parameters()
@@ -383,7 +396,10 @@ class NemotronModel(TextModel):
rot_pct = self.rope_parameters["partial_rotary_factor"]
n_embd = self.find_hparam(["hidden_size", "n_embd"])
n_head = self.find_hparam(["num_attention_heads", "n_head"])
- self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head)
+ # Use explicit head_dim when available; fallback keeps standard Nemotron working.
+ # Without this, models where n_head * head_dim != n_embd (e.g. 48 heads, head_dim=128) produce the wrong rope dimension count.
+ head_dim = self.hparams.get("head_dim") or n_embd // n_head
+ self.gguf_writer.add_rope_dimension_count(int(rot_pct * head_dim))
# * RopeScaling for Nemotron
factor = self.hparams.get("factor") or self.rope_parameters.get("factor")
diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py
index 7d8c4fda6..37df033ef 100755
--- a/convert_hf_to_gguf_update.py
+++ b/convert_hf_to_gguf_update.py
@@ -178,6 +178,7 @@ pre_computed_hashes = [
{"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.5-Air", "chkhsh": "9ca2dd618e8afaf09731a7cf6e2105b373ba6a1821559f258b272fe83e6eb902"},
{"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.7-Flash", "chkhsh": "cdf5f35325780597efd76153d4d1c16778f766173908894c04afc20108536267"},
{"name": "minerva-7b", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0", "chkhsh": "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35"},
+ {"name": "domyn-small", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/domyn/Domyn-Small-v1.0", "chkhsh": "6e4d72c6e5260b90606e26a0931fa15e4b42ec6e97719d12919f2b1f93478262"},
{"name": "hunyuan", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-A13B-Instruct", "chkhsh": "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664"},
{"name": "hunyuan-dense", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-4B-Instruct", "chkhsh": "bba3b3366b646dbdded5dbc42d59598b849371afc42f7beafa914afaa5b70aa6"},
{"name": "hy_v4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hy4-preview", "chkhsh": "e6ddf9c6686791c12d698d34c31ab9be1fea9af5a3d9a6909783ab382198ae1c"},
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 4482f7f33..0b95580bd 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -383,6 +383,8 @@ class Keys:
NORMALIZER_NFC = "tokenizer.ggml.normalizer.nfc"
NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"
NORMALIZER_STRIP_ACCENTS = "tokenizer.ggml.normalizer.strip_accents"
+ # Metaspace constants
+ METASPACE_PREPEND_SCHEME = "tokenizer.ggml.metaspace.prepend_scheme"
# FIM/Infill special tokens constants
FIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id"
FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id"
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 2faaefa59..9e49a1514 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1377,6 +1377,9 @@ class GGUFWriter:
def add_normalizer_strip_accents(self, value: bool) -> None:
self.add_bool(Keys.Tokenizer.NORMALIZER_STRIP_ACCENTS, value)
+ def add_metaspace_prepend_scheme(self, value: str) -> None:
+ self.add_string(Keys.Tokenizer.METASPACE_PREPEND_SCHEME, value)
+
def add_eot_token_id(self, id: int) -> None:
self.add_uint32(Keys.Tokenizer.EOT_ID, id)
diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py
index 6383178ea..3b82c3961 100644
--- a/gguf-py/gguf/vocab.py
+++ b/gguf-py/gguf/vocab.py
@@ -56,6 +56,7 @@ class SpecialVocab:
normalizer_lowercase: bool | None
normalizer_strip_accents: bool | None
clean_spaces: bool | None
+ metaspace_prepend_scheme: str | None
def __init__(
self, path: str | os.PathLike[str], load_merges: bool = False,
@@ -72,6 +73,7 @@ class SpecialVocab:
self.normalizer_lowercase = None
self.normalizer_strip_accents = None
self.clean_spaces = None
+ self.metaspace_prepend_scheme = None
if special_token_types is not None:
self.special_token_types = special_token_types
else:
@@ -126,6 +128,10 @@ class SpecialVocab:
if not quiet:
logger.info(f'Setting clean_spaces to {self.clean_spaces}')
gw.add_clean_spaces(self.clean_spaces)
+ if self.metaspace_prepend_scheme is not None:
+ if not quiet:
+ logger.info(f'Setting metaspace_prepend_scheme to {self.metaspace_prepend_scheme}')
+ gw.add_metaspace_prepend_scheme(self.metaspace_prepend_scheme)
def _load(self, path: Path) -> None:
self._try_load_from_tokenizer_json(path)
@@ -194,6 +200,16 @@ class SpecialVocab:
for norm in normalizer.get('normalizers', []):
self._parse_normalizer(norm)
+ def _parse_pre_tokenizer(self, pre_tokenizer: dict) -> None:
+ # ref: https://huggingface.co/docs/tokenizers/api/pre-tokenizers
+ pre_tokenizer_type = pre_tokenizer.get('type')
+ if pre_tokenizer_type == 'Metaspace':
+ if 'prepend_scheme' in pre_tokenizer:
+ self.metaspace_prepend_scheme = pre_tokenizer['prepend_scheme']
+ elif pre_tokenizer_type == 'Sequence':
+ for pre in pre_tokenizer.get('pretokenizers', []):
+ self._parse_pre_tokenizer(pre)
+
def _try_load_from_tokenizer_json(self, path: Path) -> bool:
tokenizer = None
tokenizer_file = path / 'tokenizer.json'
@@ -229,6 +245,8 @@ class SpecialVocab:
# Parse normalizer configuration (e.g. Lowercase) into metadata
if normalizer := tokenizer.get('normalizer'):
self._parse_normalizer(normalizer)
+ if pre_tokenizer := tokenizer.get('pre_tokenizer'):
+ self._parse_pre_tokenizer(pre_tokenizer)
added_tokens = tokenizer.get('added_tokens', {})
else:
added_tokens = {}
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index d79b18dec..a364174c1 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -445,6 +445,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_TOKENIZER_NORMALIZER_NFC, "tokenizer.ggml.normalizer.nfc" },
{ LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, "tokenizer.ggml.normalizer.lowercase" },
{ LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, "tokenizer.ggml.normalizer.strip_accents" },
+ { LLM_KV_TOKENIZER_METASPACE_PREPEND_SCHEME, "tokenizer.ggml.metaspace.prepend_scheme" },
{ LLM_KV_TOKENIZER_FIM_PRE_ID, "tokenizer.ggml.fim_pre_token_id" },
{ LLM_KV_TOKENIZER_FIM_SUF_ID, "tokenizer.ggml.fim_suf_token_id" },
{ LLM_KV_TOKENIZER_FIM_MID_ID, "tokenizer.ggml.fim_mid_token_id" },
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 12d8fe392..cdf978fb8 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -408,6 +408,7 @@ enum llm_kv {
LLM_KV_TOKENIZER_NORMALIZER_NFC,
LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE,
LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS,
+ LLM_KV_TOKENIZER_METASPACE_PREPEND_SCHEME,
LLM_KV_TOKENIZER_FIM_PRE_ID,
LLM_KV_TOKENIZER_FIM_SUF_ID,
LLM_KV_TOKENIZER_FIM_MID_ID,
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 80a5fcdff..78512bef5 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -543,6 +543,7 @@ struct llm_tokenizer_bpe : llm_tokenizer {
byte_encode = false; // uses raw UTF-8, not GPT-2 byte encoding
break;
case LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE:
+ case LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL:
// Sarvam uses SPM-style BPE (same shape as Gemma4): spaces replaced with U+2581
// by the normalizer, BPE merges over the whole text on raw UTF-8.
regex_exprs = {
@@ -2097,6 +2098,8 @@ struct llama_vocab::impl {
// Tokenizer normalizer options
llama_vocab::normalizer_options normalizer_opts;
+ llama_metaspace_prepend_scheme metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_NEVER;
+
std::unordered_map<std::string, llama_token> token_to_id;
std::vector<token_data> id_to_token;
@@ -2612,6 +2615,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
} else if (
tokenizer_pre == "minerva-7b") {
pre_type = LLAMA_VOCAB_PRE_TYPE_MINERVA;
+ } else if (
+ tokenizer_pre == "domyn-small") {
+ pre_type = LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL;
+ escape_whitespaces = true;
} else if (
tokenizer_pre == "megrez") {
pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN2;
@@ -2956,6 +2963,24 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
normalizer_opts.strip_accents = normalizer_opts.lowercase;
ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, normalizer_opts.strip_accents, false);
+ {
+ std::string scheme;
+ if (ml.get_key(LLM_KV_TOKENIZER_METASPACE_PREPEND_SCHEME, scheme, false)) {
+ if (scheme == "never") {
+ metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_NEVER;
+ } else if (scheme == "first") {
+ metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_FIRST;
+ } else if (scheme == "always") {
+ metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_ALWAYS;
+ } else {
+ throw std::runtime_error(format("unknown metaspace prepend_scheme: '%s'", scheme.c_str()));
+ }
+ if (type == LLAMA_VOCAB_TYPE_BPE && metaspace_prepend_scheme == LLAMA_METASPACE_PREPEND_SCHEME_ALWAYS) {
+ throw std::runtime_error("metaspace prepend_scheme 'always' is not supported for BPE vocab");
+ }
+ }
+ }
+
// suppress tokens
{
const int suppress_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SUPPRESS_TOKENS).c_str());
@@ -3743,6 +3768,7 @@ std::vector<llama_token> llama_vocab::impl::tokenize(
// tokenizer.encode('', add_special_tokens=True) returns [1]
// tokenizer.encode('', add_special_tokens=False) returns []
+ // TODO: honor metaspace_prepend_scheme (add_space_prefix currently behaves like "always")
bool is_prev_special = true; // prefix with space if first token
if (add_special && add_bos) {
@@ -3813,6 +3839,13 @@ std::vector<llama_token> llama_vocab::impl::tokenize(
llama_escape_whitespace(text);
}
+ // Metaspace prepend_scheme "first": add U+2581 only at the start of the input
+ static const std::string escaped_space = "\xE2\x96\x81";
+ if (metaspace_prepend_scheme == LLAMA_METASPACE_PREPEND_SCHEME_FIRST &&
+ &fragment == &fragment_buffer.front() && text.rfind(escaped_space, 0) != 0) {
+ text.insert(0, escaped_space);
+ }
+
#ifdef PRETOKENIZERDEBUG
LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", text.length(), fragment.offset, fragment.length, text.c_str());
#endif
@@ -3860,6 +3893,7 @@ std::vector<llama_token> llama_vocab::impl::tokenize(
GGML_ASSERT(special_bos_id != LLAMA_TOKEN_NULL);
output.push_back(special_bos_id);
}
+ // TODO: honor metaspace_prepend_scheme
llm_tokenizer_ugm_session session(vocab, *static_cast<const llm_tokenizer_ugm *>(tokenizer.get()));
for (const auto & fragment : fragment_buffer) {
@@ -4143,10 +4177,17 @@ int32_t llama_vocab::impl::detokenize(
}
}
+ // Metaspace prepend_scheme "first": remove the leading space of the first rendered piece, so a skipped BOS does not consume it
+ // TODO: honor metaspace_prepend_scheme for the other vocab types
+ bool remove_first_space = type == LLAMA_VOCAB_TYPE_BPE && metaspace_prepend_scheme == LLAMA_METASPACE_PREPEND_SCHEME_FIRST;
+
for (int32_t i = 0; i < n_tokens; ++i) {
GGML_ASSERT(avail >= 0);
- int32_t n_chars = token_to_piece(tokens[i], text, avail, remove_space, unparse_special);
+ int32_t n_chars = token_to_piece(tokens[i], text, avail, remove_space || remove_first_space, unparse_special);
remove_space = false;
+ if (unparse_special || !(token_get_attr(tokens[i]) & (LLAMA_TOKEN_ATTR_UNKNOWN | LLAMA_TOKEN_ATTR_CONTROL))) {
+ remove_first_space = false;
+ }
if (n_chars < 0) {
avail = 0;
total -= n_chars;
diff --git a/src/llama-vocab.h b/src/llama-vocab.h
index a87c0de42..40357dba7 100644
--- a/src/llama-vocab.h
+++ b/src/llama-vocab.h
@@ -71,6 +71,14 @@ enum llama_vocab_pre_type {
LLAMA_VOCAB_PRE_TYPE_MMBERT = 60,
LLAMA_VOCAB_PRE_TYPE_K2_HORIZON = 61,
LLAMA_VOCAB_PRE_TYPE_FLINT = 62,
+ LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL = 63,
+};
+
+// HF Metaspace pre-tokenizer prepend_scheme
+enum llama_metaspace_prepend_scheme {
+ LLAMA_METASPACE_PREPEND_SCHEME_NEVER = 0,
+ LLAMA_METASPACE_PREPEND_SCHEME_FIRST = 1,
+ LLAMA_METASPACE_PREPEND_SCHEME_ALWAYS = 2,
};
struct LLM_KV;
diff --git a/src/models/nemotron.cpp b/src/models/nemotron.cpp
index 129cf622e..6b5b77107 100644
--- a/src/models/nemotron.cpp
+++ b/src/models/nemotron.cpp
@@ -25,8 +25,11 @@ void llama_model_nemotron::load_arch_tensors(llama_model_loader &) {
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
- create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
- layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
+ // n_embd_head_k * n_head equals n_embd for standard Nemotron (32 heads),
+ // but differs for models with non-standard GQA (e.g. 48 Q heads, head_dim=128).
+ const int64_t n_embd_q = n_embd_head_k * n_head;
+ create_tensor_qkv(layer, i, n_embd, n_embd_q, n_embd_gqa, n_embd_gqa, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0);
// optional bias tensors
layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);