Commit 582eab718 for llama.cpp
commit 582eab718608b99792286729890aac1675547c29
Author: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
Date: Tue Nov 10 21:54:14 2026 +0100
ci : bump ty to 0.0.100 (#32030)
* bump ty to 0.0.100
* rename rules
* type fixes
* type fix
* proper type fix
diff --git a/.github/workflows/python-type-check.yml b/.github/workflows/python-type-check.yml
index 8fb4ffd47..f4ba40fc2 100644
--- a/.github/workflows/python-type-check.yml
+++ b/.github/workflows/python-type-check.yml
@@ -35,7 +35,7 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: "3.11"
- pip-install: -r requirements/requirements-all.txt ty==0.0.88
+ pip-install: -r requirements/requirements-all.txt ty==0.0.100
# - name: Type-check with Pyright
# uses: jakebailey/pyright-action@v2
# with:
diff --git a/conversion/base.py b/conversion/base.py
index 6dee2e878..58ddb5def 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -32,10 +32,10 @@ import gguf
from gguf.vocab import MistralTokenizerType, MistralVocab
try:
- from mistral_common.tokens.tokenizers.base import TokenizerVersion # type: ignore[import-not-found, ty:unresolved-import]
- from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN as _MISTRAL_COMMON_DATASET_MEAN, DATASET_STD as _MISTRAL_COMMON_DATASET_STD # type: ignore[import-not-found, ty:unresolved-import]
- from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import]
- from mistral_common.tokens.tokenizers.sentencepiece import ( # type: ignore[import-not-found, ty:unresolved-import]
+ from mistral_common.tokens.tokenizers.base import TokenizerVersion # ty: ignore[import-not-found]
+ from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN as _MISTRAL_COMMON_DATASET_MEAN, DATASET_STD as _MISTRAL_COMMON_DATASET_STD # ty: ignore[import-not-found]
+ from mistral_common.tokens.tokenizers.tekken import Tekkenizer # ty: ignore[import-not-found]
+ from mistral_common.tokens.tokenizers.sentencepiece import ( # ty: ignore[import-not-found]
SentencePieceTokenizer,
)
@@ -273,7 +273,7 @@ class ModelBase:
if weight_map is None or not isinstance(weight_map, dict):
raise ValueError(f"Can't load 'weight_map' from {index_name!r}")
tensor_names_from_index.update(weight_map.keys())
- part_dict: dict[str, None] = dict.fromkeys(weight_map.values(), None) # ty: ignore[invalid-assignment]
+ part_dict: dict[str, None] = dict.fromkeys(weight_map.values(), None)
part_names = sorted(part_dict.keys())
else:
weight_map = {}
@@ -2055,15 +2055,16 @@ class TextModel(ModelBase):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
- vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) # ty: ignore[unresolved-attribute]
- assert max(tokenizer.vocab.values()) < vocab_size # ty: ignore[unresolved-attribute]
+ assert tokenizer is not None
+ vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))
+ assert max(tokenizer.vocab.values()) < vocab_size
tokpre = self.get_vocab_base_pre(tokenizer)
- reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} # ty: ignore[unresolved-attribute]
- added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
+ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
+ added_vocab = tokenizer.get_added_vocab()
- added_tokens_decoder = tokenizer.added_tokens_decoder # ty: ignore[unresolved-attribute]
+ added_tokens_decoder = tokenizer.added_tokens_decoder
for i in range(vocab_size):
if i not in reverse_vocab:
@@ -2076,7 +2077,7 @@ class TextModel(ModelBase):
# To avoid unexpected issues - we make sure to normalize non-normalized tokens
if not added_tokens_decoder[i].normalized:
previous_token = token
- token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[unresolved-attribute, invalid-assignment]
+ token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[invalid-assignment]
if previous_token != token:
logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
@@ -2486,6 +2487,7 @@ class TextModel(ModelBase):
def _set_vocab_hybriddna(self):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+ assert tokenizer is not None
vocab = tokenizer.get_vocab()
vocab_size = self.hparams.get("vocab_size", len(vocab))
assert max(vocab.values()) < vocab_size
@@ -2499,11 +2501,11 @@ class TextModel(ModelBase):
if tokenizer.k != 6 or tail not in ("pad_a", "pad_n"):
raise NotImplementedError(f"HybridDNATokenizer with k={tokenizer.k}, tail={tail!r}")
for kmer in [*tokenizer.kmers, *getattr(tokenizer, "npad_tokens", [])]:
- reverse_vocab[tokenizer.dna_token_to_id[kmer]] = kmer + "\ue000" # ty: ignore[unresolved-attribute]
+ reverse_vocab[tokenizer.dna_token_to_id[kmer]] = kmer + "\ue000"
# older HybridDNATokenizer versions (Carbon-500M/3B/8B) do not register the base tokenizer's added tokens (<|im_start|>, <think>, ...) as their own
# but they are still encoded as special tokens: take their types from the base tokenizer
base_tokenizer = getattr(tokenizer, "_base_tokenizer", tokenizer)
- added_tokens_decoder = {**base_tokenizer.added_tokens_decoder, **tokenizer.added_tokens_decoder} # ty: ignore[unresolved-attribute]
+ added_tokens_decoder = {**base_tokenizer.added_tokens_decoder, **tokenizer.added_tokens_decoder}
tokens: list[str] = []
toktypes: list[int] = []
@@ -2550,24 +2552,25 @@ class TextModel(ModelBase):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
+ assert tokenizer is not None
vocab_size = hparams["vocab_size"]
- assert max(tokenizer.get_vocab().values()) < vocab_size # ty: ignore[unresolved-attribute]
+ assert max(tokenizer.get_vocab().values()) < vocab_size
tokpre = self.get_vocab_base_pre(tokenizer)
merges = []
vocab = {}
- mergeable_ranks = tokenizer.mergeable_ranks # ty: ignore[unresolved-attribute]
+ mergeable_ranks = tokenizer.mergeable_ranks
for token, rank in mergeable_ranks.items():
vocab[QwenModel.token_bytes_to_string(token)] = rank
if len(token) == 1:
continue
merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)
assert len(merged) == 2
- merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))
+ merges.append(str(' '.join(map(QwenModel.token_bytes_to_string, merged))))
# for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combined
- added_vocab = tokenizer.special_tokens # ty: ignore[unresolved-attribute]
+ added_vocab = tokenizer.special_tokens
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **added_vocab}.items()}
for i in range(vocab_size):
@@ -2590,10 +2593,10 @@ class TextModel(ModelBase):
special_vocab.merges = merges
# only add special tokens when they were not already loaded from config.json
if len(special_vocab.special_token_ids) == 0:
- special_vocab._set_special_token("bos", tokenizer.special_tokens["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
- special_vocab._set_special_token("eos", tokenizer.special_tokens["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
+ special_vocab._set_special_token("bos", tokenizer.special_tokens["<|endoftext|>"])
+ special_vocab._set_special_token("eos", tokenizer.special_tokens["<|endoftext|>"])
# this one is usually not in config.json anyway
- special_vocab._set_special_token("unk", tokenizer.special_tokens["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
+ special_vocab._set_special_token("unk", tokenizer.special_tokens["<|endoftext|>"])
special_vocab.add_to_gguf(self.gguf_writer)
def _set_vocab_sentencepiece(self, add_to_gguf=True):
@@ -2867,21 +2870,23 @@ class TextModel(ModelBase):
def _set_vocab_glmedge(self):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+ assert tokenizer is not None
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
tokens, toktypes, tokpre = self.get_vocab_base()
self.gguf_writer.add_tokenizer_model("gpt2")
self.gguf_writer.add_tokenizer_pre(tokpre)
self.gguf_writer.add_token_list(tokens)
self.gguf_writer.add_token_types(toktypes)
- special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
- special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # ty: ignore[unresolved-attribute]
- special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
- special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
+ special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
+ special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])
+ special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])
+ special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"])
special_vocab.add_to_gguf(self.gguf_writer)
def _set_vocab_glm(self):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+ assert tokenizer is not None
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
tokens, toktypes, tokpre = self.get_vocab_base()
self.gguf_writer.add_tokenizer_model("gpt2")
@@ -2890,10 +2895,10 @@ class TextModel(ModelBase):
self.gguf_writer.add_token_types(toktypes)
# Special tokens
# Note: Using <|endoftext|> (151329) for eot causes endless generation
- special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"]) # ty: ignore[unresolved-attribute] # 151331
- special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # ty: ignore[unresolved-attribute] # 151336
- special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute] # 151329
- special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # ty: ignore[unresolved-attribute] # 151338
+ special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"]) # 151331
+ special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # 151336
+ special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # 151329
+ special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"]) # 151338
special_vocab.add_to_gguf(self.gguf_writer)
def _set_vocab_interns1(self):
@@ -2902,16 +2907,17 @@ class TextModel(ModelBase):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
- vocab = getattr(tokenizer, 'vocab', tokenizer.get_vocab()) # ty: ignore[unresolved-attribute]
+ assert tokenizer is not None
+ vocab = getattr(tokenizer, 'vocab', tokenizer.get_vocab())
vocab_size = self.hparams.get("vocab_size", len(vocab))
assert max(vocab.values()) < vocab_size
tokpre = self.get_vocab_base_pre(tokenizer)
reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab.items()}
- added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
+ added_vocab = tokenizer.get_added_vocab()
- added_tokens_decoder = tokenizer.added_tokens_decoder # ty: ignore[unresolved-attribute]
+ added_tokens_decoder = tokenizer.added_tokens_decoder
for i in range(vocab_size):
if i not in reverse_vocab:
@@ -2924,7 +2930,7 @@ class TextModel(ModelBase):
# To avoid unexpected issues - we make sure to normalize non-normalized tokens
if not added_tokens_decoder[i].normalized:
previous_token = token
- token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[unresolved-attribute, invalid-assignment]
+ token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[invalid-assignment]
if previous_token != token:
logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
diff --git a/conversion/beam.py b/conversion/beam.py
index 67d0cc93c..66b28d8f6 100644
--- a/conversion/beam.py
+++ b/conversion/beam.py
@@ -60,7 +60,8 @@ class BeamModel(TextModel):
# otherwise EOT is guessed from a hash-ordered scan
from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained(self.dir_model)
- ret_id = tok.convert_tokens_to_ids("<|return|>") # ty: ignore[unresolved-attribute]
+ assert tok is not None
+ ret_id = tok.convert_tokens_to_ids("<|return|>")
if isinstance(ret_id, int) and ret_id >= 0:
self.gguf_writer.add_eot_token_id(ret_id)
diff --git a/conversion/bert.py b/conversion/bert.py
index f24715426..150b2667c 100644
--- a/conversion/bert.py
+++ b/conversion/bert.py
@@ -132,6 +132,7 @@ class BertModel(TextModel):
from base64 import b64decode
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+ assert tokenizer is not None
with open(tokenizer_path, "r", encoding="utf-8") as fp:
tokenizer_json = json.load(fp)
@@ -140,13 +141,13 @@ class BertModel(TextModel):
with open(tokenizer_config_path, "r", encoding="utf-8") as fp:
tokenizer_config_json = json.load(fp)
- add_prefix = tokenizer.add_prefix_space # ty: ignore[unresolved-attribute]
- remove_whitespaces = tokenizer.clean_up_tokenization_spaces # ty: ignore[unresolved-attribute]
+ add_prefix = tokenizer.add_prefix_space
+ remove_whitespaces = tokenizer.clean_up_tokenization_spaces
precompiled_charsmap = b64decode(tokenizer_json["normalizer"]["precompiled_charsmap"])
- vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size) # ty: ignore[unresolved-attribute]
+ vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size)
else:
- sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+ sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
assert sentencepiece_model.trainer_spec.model_type == 1 # UNIGRAM
@@ -183,20 +184,20 @@ class BertModel(TextModel):
scores[token_id] = score
toktypes[token_id] = toktype
else:
- added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
+ added_vocab = tokenizer.get_added_vocab()
unk_token = tokenizer_config_json.get("unk_token")
- unk_token_id = added_vocab.get(unk_token, tokenizer_json["model"].get("unk_id", 3)) # ty: ignore[no-matching-overload]
+ unk_token_id = added_vocab.get(unk_token, tokenizer_json["model"].get("unk_id", 3))
- for token_id in range(tokenizer.vocab_size): # ty: ignore[unresolved-attribute]
- piece = tokenizer._convert_id_to_token(token_id) # ty: ignore[unresolved-attribute]
- if (piece := tokenizer._convert_id_to_token(token_id)) is not None: # ty: ignore[unresolved-attribute]
+ for token_id in range(tokenizer.vocab_size):
+ piece = tokenizer._convert_id_to_token(token_id)
+ if (piece := tokenizer._convert_id_to_token(token_id)) is not None:
text = piece.encode("utf-8")
score = tokenizer_json["model"]["vocab"][token_id][1]
toktype = SentencePieceTokenTypes.NORMAL
if token_id == unk_token_id:
toktype = SentencePieceTokenTypes.UNKNOWN
- elif token_id in tokenizer.all_special_ids: # ty: ignore[unresolved-attribute]
+ elif token_id in tokenizer.all_special_ids:
toktype = SentencePieceTokenTypes.CONTROL
elif token_id in added_vocab.values():
toktype = SentencePieceTokenTypes.USER_DEFINED
diff --git a/conversion/chatglm.py b/conversion/chatglm.py
index 9b902dae3..fb10393cb 100644
--- a/conversion/chatglm.py
+++ b/conversion/chatglm.py
@@ -22,12 +22,13 @@ class ChatGLMModel(TextModel):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
- vocab_size = hparams.get("padded_vocab_size", len(tokenizer.get_vocab())) # ty: ignore[unresolved-attribute]
- assert max(tokenizer.get_vocab().values()) < vocab_size # ty: ignore[unresolved-attribute]
+ assert tokenizer is not None
+ vocab_size = hparams.get("padded_vocab_size", len(tokenizer.get_vocab()))
+ assert max(tokenizer.get_vocab().values()) < vocab_size
role_special_tokens = ["<|system|>", "<|user|>", "<|assistant|>", "<|observation|>"]
special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "sop", "eop"] + role_special_tokens
for token_id in range(vocab_size):
- piece = tokenizer._convert_id_to_token(token_id) # ty: ignore[unresolved-attribute]
+ piece = tokenizer._convert_id_to_token(token_id)
if token_id == 0:
piece = "<unk>"
elif token_id == 1:
@@ -35,17 +36,18 @@ class ChatGLMModel(TextModel):
elif token_id == 2:
piece = "<eos>"
- text = piece.encode("utf-8") # ty: ignore[unresolved-attribute]
+ assert isinstance(piece, str)
+ text = piece.encode("utf-8")
score = 0.0
# Referencing the tokenizer Python implementation(https://huggingface.co/THUDM/chatglm3-6b/blob/main/tokenization_chatglm.py),
# it is only valid if it is less than tokenizer.tokenizer.sp_model.vocab_size()
- if len(piece) != 0 and token_id < tokenizer.tokenizer.sp_model.vocab_size(): # ty: ignore[unresolved-attribute, invalid-argument-type]
- score = tokenizer.tokenizer.sp_model.get_score(token_id) # ty: ignore[unresolved-attribute]
+ if len(piece) != 0 and token_id < tokenizer.tokenizer.sp_model.vocab_size():
+ score = tokenizer.tokenizer.sp_model.get_score(token_id)
- if token_id >= tokenizer.tokenizer.sp_model.vocab_size(): # ty: ignore[unresolved-attribute]
+ if token_id >= tokenizer.tokenizer.sp_model.vocab_size():
if piece in special_tokens:
toktype = SentencePieceTokenTypes.CONTROL
- elif len(piece) == 0: # ty: ignore[invalid-argument-type]
+ elif len(piece) == 0:
text = f"[PAD{token_id}]".encode("utf-8")
toktype = SentencePieceTokenTypes.UNUSED
else:
@@ -56,13 +58,13 @@ class ChatGLMModel(TextModel):
continue
toktype = SentencePieceTokenTypes.NORMAL
- if tokenizer.tokenizer.sp_model.is_unknown(token_id): # ty: ignore[unresolved-attribute]
+ if tokenizer.tokenizer.sp_model.is_unknown(token_id):
toktype = SentencePieceTokenTypes.UNKNOWN
- elif tokenizer.tokenizer.sp_model.is_control(token_id): # ty: ignore[unresolved-attribute]
+ elif tokenizer.tokenizer.sp_model.is_control(token_id):
toktype = SentencePieceTokenTypes.CONTROL
- elif tokenizer.tokenizer.sp_model.is_unused(token_id): # ty: ignore[unresolved-attribute]
+ elif tokenizer.tokenizer.sp_model.is_unused(token_id):
toktype = SentencePieceTokenTypes.UNUSED
- elif tokenizer.tokenizer.sp_model.is_byte(token_id): # ty: ignore[unresolved-attribute]
+ elif tokenizer.tokenizer.sp_model.is_byte(token_id):
toktype = SentencePieceTokenTypes.BYTE
tokens.append(text)
@@ -115,8 +117,9 @@ class ChatGLMModel(TextModel):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
+ assert tokenizer is not None
vocab_size = hparams.get("padded_vocab_size",hparams["vocab_size"])
- assert max(tokenizer.get_vocab().values()) < vocab_size # ty: ignore[unresolved-attribute]
+ assert max(tokenizer.get_vocab().values()) < vocab_size
tokens, toktypes, tokpre = self.get_vocab_base()
self.gguf_writer.add_tokenizer_model("gpt2")
@@ -125,10 +128,10 @@ class ChatGLMModel(TextModel):
self.gguf_writer.add_token_types(toktypes)
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
# only add special tokens when they were not already loaded from config.json
- special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
- special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # ty: ignore[unresolved-attribute]
+ special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
+ special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])
# this one is usually not in config.json anyway
- special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
+ special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])
special_vocab.add_to_gguf(self.gguf_writer)
def set_gguf_parameters(self):
diff --git a/conversion/deepseek.py b/conversion/deepseek.py
index d559b604e..733a5e5f9 100644
--- a/conversion/deepseek.py
+++ b/conversion/deepseek.py
@@ -277,13 +277,14 @@ class DeepseekV2Model(TextModel):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+ assert tokenizer is not None
tokpre = self.get_vocab_base_pre(tokenizer)
if tokpre == "kimi-k2":
# Build merges list using the approach similar to HunYuanMoE
merges = []
vocab = {}
- mergeable_ranks = tokenizer.model._mergeable_ranks # ty: ignore[unresolved-attribute]
+ mergeable_ranks = tokenizer.model._mergeable_ranks # ty: ignore[missing-union-attribute]
for token, rank in mergeable_ranks.items():
vocab[QwenModel.token_bytes_to_string(token)] = rank
if len(token) == 1:
@@ -294,7 +295,7 @@ class DeepseekV2Model(TextModel):
# Build token list
vocab_size = self.hparams["vocab_size"]
- special_tokens = tokenizer.special_tokens # ty: ignore[unresolved-attribute]
+ special_tokens = tokenizer.special_tokens
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
tokens: list[str] = []
toktypes: list[int] = []
diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py
index 6b098a037..9a03b9912 100644
--- a/conversion/deepseek41.py
+++ b/conversion/deepseek41.py
@@ -62,6 +62,7 @@ def engram_compressed_token_map(tokenizer_dir: Path) -> tuple[list[int], int]:
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir)
+ assert tokenizer is not None
sentinel = "\ue000"
normalizer = normalizers.Sequence([
normalizers.NFKC(),
@@ -80,6 +81,7 @@ def engram_compressed_token_map(tokenizer_dir: Path) -> tuple[list[int], int]:
text = backend.decode([token_id], skip_special_tokens=False)
if "\ufffd" in text:
key = backend.id_to_token(token_id)
+ assert key is not None
else:
normalized = normalizer.normalize_str(text)
key = normalized if normalized else text
diff --git a/conversion/dots3.py b/conversion/dots3.py
index c7ac2319e..e34793e58 100644
--- a/conversion/dots3.py
+++ b/conversion/dots3.py
@@ -75,13 +75,14 @@ class Dots3NoteModel(DeepseekV2Model):
def set_vocab(self):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+ assert tokenizer is not None
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
tokens, toktypes, tokpre = self.get_vocab_base()
self.gguf_writer.add_tokenizer_model("gpt2")
self.gguf_writer.add_tokenizer_pre(tokpre)
self.gguf_writer.add_token_list(tokens)
self.gguf_writer.add_token_types(toktypes)
- special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endofassistant|>"]) # ty: ignore[unresolved-attribute]
+ special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endofassistant|>"])
special_vocab.add_to_gguf(self.gguf_writer)
@classmethod
diff --git a/conversion/dream.py b/conversion/dream.py
index 14f25404d..46c245eee 100644
--- a/conversion/dream.py
+++ b/conversion/dream.py
@@ -19,15 +19,16 @@ class DreamModel(TextModel):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+ assert tokenizer is not None
- vocab_dict = tokenizer.get_vocab() # ty: ignore[unresolved-attribute]
+ vocab_dict = tokenizer.get_vocab()
vocab_size = self.hparams.get("vocab_size", len(vocab_dict))
assert max(vocab_dict.values()) < vocab_size
tokpre = self.get_vocab_base_pre(tokenizer)
reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab_dict.items()}
- added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
+ added_vocab = tokenizer.get_added_vocab()
for i in range(vocab_size):
if i not in reverse_vocab:
diff --git a/conversion/glm.py b/conversion/glm.py
index 10f3ef113..4aa2288de 100644
--- a/conversion/glm.py
+++ b/conversion/glm.py
@@ -31,6 +31,7 @@ class Glm4Model(TextModel):
def set_vocab(self):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+ assert tokenizer is not None
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
tokens, toktypes, tokpre = self.get_vocab_base()
self.gguf_writer.add_tokenizer_model("gpt2")
@@ -38,10 +39,10 @@ class Glm4Model(TextModel):
self.gguf_writer.add_token_list(tokens)
self.gguf_writer.add_token_types(toktypes)
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
- special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
- special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"]) # ty: ignore[unresolved-attribute]
- special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
- special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
+ special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
+ special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])
+ special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])
+ special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"])
special_vocab.add_to_gguf(self.gguf_writer)
def set_gguf_parameters(self):
@@ -391,16 +392,17 @@ class SolarOpenModel(Glm4MoeModel):
def set_vocab(self):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+ assert tokenizer is not None
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
tokens, toktypes, tokpre = self.get_vocab_base()
self.gguf_writer.add_tokenizer_model("gpt2")
self.gguf_writer.add_tokenizer_pre(tokpre)
self.gguf_writer.add_token_list(tokens)
self.gguf_writer.add_token_types(toktypes)
- special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
- special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endoftext|>"]) # ty: ignore[unresolved-attribute]
- special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<unk>"]) # ty: ignore[unresolved-attribute]
- special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|startoftext|>"]) # ty: ignore[unresolved-attribute]
+ special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
+ special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endoftext|>"])
+ special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<unk>"])
+ special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|startoftext|>"])
special_vocab.add_to_gguf(self.gguf_writer)
diff --git a/conversion/hunyuan.py b/conversion/hunyuan.py
index 58dee2ec8..2088953c1 100644
--- a/conversion/hunyuan.py
+++ b/conversion/hunyuan.py
@@ -24,6 +24,7 @@ class HunYuanMoEModel(TextModel):
def set_vocab(self):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+ assert tokenizer is not None
# 1. Get the pre-tokenizer identifier hash
tokpre = self.get_vocab_base_pre(tokenizer)
@@ -31,7 +32,7 @@ class HunYuanMoEModel(TextModel):
# 2. Reverse-engineer the merges list from mergeable_ranks
merges = []
vocab = {}
- mergeable_ranks = tokenizer.mergeable_ranks # ty: ignore[unresolved-attribute]
+ mergeable_ranks = tokenizer.mergeable_ranks
for token, rank in mergeable_ranks.items():
vocab[QwenModel.token_bytes_to_string(token)] = rank
if len(token) == 1:
@@ -42,8 +43,8 @@ class HunYuanMoEModel(TextModel):
# 3. Generate the tokens and toktypes lists
vocab_size = self.hparams["vocab_size"]
- assert tokenizer.vocab_size == vocab_size # ty: ignore[unresolved-attribute]
- special_tokens = tokenizer.special_tokens # ty: ignore[unresolved-attribute]
+ assert tokenizer.vocab_size == vocab_size
+ special_tokens = tokenizer.special_tokens
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
tokens: list[str] = []
toktypes: list[int] = []
@@ -184,6 +185,7 @@ class HunYuanModel(TextModel):
else:
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+ assert tokenizer is not None
# 1. Get the pre-tokenizer identifier hash
tokpre = self.get_vocab_base_pre(tokenizer)
@@ -191,7 +193,7 @@ class HunYuanModel(TextModel):
# 2. Reverse-engineer the merges list from mergeable_ranks
merges = []
vocab = {}
- mergeable_ranks = tokenizer.mergeable_ranks # ty: ignore[unresolved-attribute]
+ mergeable_ranks = tokenizer.mergeable_ranks
for token, rank in mergeable_ranks.items():
vocab[QwenModel.token_bytes_to_string(token)] = rank
if len(token) == 1:
@@ -202,8 +204,8 @@ class HunYuanModel(TextModel):
# 3. Generate the tokens and toktypes lists
vocab_size = self.hparams["vocab_size"]
- assert tokenizer.vocab_size == vocab_size # ty: ignore[unresolved-attribute]
- special_tokens = tokenizer.special_tokens # ty: ignore[unresolved-attribute]
+ assert tokenizer.vocab_size == vocab_size
+ special_tokens = tokenizer.special_tokens
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
tokens: list[str] = []
toktypes: list[int] = []
diff --git a/conversion/internlm.py b/conversion/internlm.py
index df2668474..b1c7bddd3 100644
--- a/conversion/internlm.py
+++ b/conversion/internlm.py
@@ -36,7 +36,7 @@ class InternLM2Model(TextModel):
logger.error(f'Error: Missing {tokenizer_path}')
sys.exit(1)
- sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+ sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix
diff --git a/conversion/kimi_linear.py b/conversion/kimi_linear.py
index 697ab1b4a..c205003a5 100644
--- a/conversion/kimi_linear.py
+++ b/conversion/kimi_linear.py
@@ -29,13 +29,14 @@ class KimiLinearModel(TextModel):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+ assert tokenizer is not None
tokpre = self.get_vocab_base_pre(tokenizer)
if tokpre == "kimi-k2":
# Build merges list using the approach similar to HunYuanMoE
merges = []
vocab = {}
- mergeable_ranks = tokenizer.model._mergeable_ranks # ty: ignore[unresolved-attribute]
+ mergeable_ranks = tokenizer.model._mergeable_ranks # ty: ignore[missing-union-attribute]
for token, rank in mergeable_ranks.items():
vocab[QwenModel.token_bytes_to_string(token)] = rank
if len(token) == 1:
@@ -45,7 +46,7 @@ class KimiLinearModel(TextModel):
merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))
# Build token list
vocab_size = self.hparams["vocab_size"]
- special_tokens = tokenizer.special_tokens # ty: ignore[unresolved-attribute]
+ special_tokens = tokenizer.special_tokens
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
tokens: list[str] = []
toktypes: list[int] = []
@@ -71,7 +72,7 @@ class KimiLinearModel(TextModel):
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)
special_vocab.add_to_gguf(self.gguf_writer)
# override eos id in config.json with tiktoken eos id
- self.gguf_writer.add_eos_token_id(tokenizer.eos_id) # ty: ignore[unresolved-attribute]
+ self.gguf_writer.add_eos_token_id(tokenizer.eos_id)
else:
raise NotImplementedError(f"Deepseek pre-tokenizer {tokpre!r} is not supported yet!")
diff --git a/conversion/lev.py b/conversion/lev.py
index 72b12445d..5c319ec25 100644
--- a/conversion/lev.py
+++ b/conversion/lev.py
@@ -77,10 +77,10 @@ class _DecisionLoraMixin:
delta = self.lora_scale * (lora["B"] @ lora["A"])
data_torch = data_torch.float() + LazyTorchTensor.from_eager(delta)
self.lora_merged.add(name[name.index("layers."):])
- yield from super().modify_tensors(data_torch, name, bid) # ty: ignore[unresolved-attribute]
+ yield from super().modify_tensors(data_torch, name, bid) # ty: ignore[missing-attribute]
def prepare_tensors(self):
- super().prepare_tensors() # ty: ignore[unresolved-attribute]
+ super().prepare_tensors() # ty: ignore[missing-attribute]
if len(self.lora_merged) != len(self.lora):
raise ValueError(f"only {len(self.lora_merged)} of {len(self.lora)} LoRA tensors were merged into the base model")
diff --git a/conversion/llada.py b/conversion/llada.py
index c03607191..7f91536e1 100644
--- a/conversion/llada.py
+++ b/conversion/llada.py
@@ -22,15 +22,16 @@ class LLaDAModel(TextModel):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+ assert tokenizer is not None
- vocab_dict = tokenizer.get_vocab() # ty: ignore[unresolved-attribute]
+ vocab_dict = tokenizer.get_vocab()
vocab_size = self.hparams.get("vocab_size", len(vocab_dict))
assert max(vocab_dict.values()) < vocab_size
tokpre = self.get_vocab_base_pre(tokenizer)
reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab_dict.items()}
- added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
+ added_vocab = tokenizer.get_added_vocab()
for i in range(vocab_size):
if i not in reverse_vocab:
diff --git a/conversion/minicpm.py b/conversion/minicpm.py
index 8c7be8a9a..2b239437c 100644
--- a/conversion/minicpm.py
+++ b/conversion/minicpm.py
@@ -263,6 +263,7 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel):
# the tile separators are text tokens; clip appends their embeddings so that one chunk holds the whole image
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+ assert tokenizer is not None
ids = []
for text in self.tok_embd_sep:
tok = tokenizer.encode(text, add_special_tokens=False)
diff --git a/conversion/minimax.py b/conversion/minimax.py
index aac340c61..4d856fab8 100644
--- a/conversion/minimax.py
+++ b/conversion/minimax.py
@@ -25,7 +25,8 @@ class MiniMaxText01Model(TextModel):
# they get in the way of the token sampling process and must be suppressed
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
- tokenizer_vocab_size = tokenizer.vocab_size # ty: ignore[unresolved-attribute]
+ assert tokenizer is not None
+ tokenizer_vocab_size = tokenizer.vocab_size
with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f:
weight_map = json.load(f)["weight_map"]
diff --git a/conversion/mistral.py b/conversion/mistral.py
index aec22ca38..b2b5b2557 100644
--- a/conversion/mistral.py
+++ b/conversion/mistral.py
@@ -12,9 +12,9 @@ from .deepseek import DeepseekV2Model
from .llama import LlamaModel
if _mistral_common_installed:
- from mistral_common.tokens.tokenizers.base import TokenizerVersion # type: ignore[import-not-found, ty:unresolved-import]
- from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import]
- from mistral_common.tokens.tokenizers.sentencepiece import SentencePieceTokenizer # type: ignore[import-not-found, ty:unresolved-import]
+ from mistral_common.tokens.tokenizers.base import TokenizerVersion # ty: ignore[import-not-found]
+ from mistral_common.tokens.tokenizers.tekken import Tekkenizer # ty: ignore[import-not-found]
+ from mistral_common.tokens.tokenizers.sentencepiece import SentencePieceTokenizer # ty: ignore[import-not-found]
else:
TokenizerVersion = None # type: ignore[assignment]
Tekkenizer = None # type: ignore[assignment]
diff --git a/conversion/muse_glimmer.py b/conversion/muse_glimmer.py
index 9b63d3dba..f00b3e6b5 100644
--- a/conversion/muse_glimmer.py
+++ b/conversion/muse_glimmer.py
@@ -37,7 +37,8 @@ class MuseGlimmerModel(TextModel):
from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained(self.dir_model)
- eot_id = tok.convert_tokens_to_ids("<|eot|>") # ty: ignore[unresolved-attribute]
+ assert tok is not None
+ eot_id = tok.convert_tokens_to_ids("<|eot|>")
if isinstance(eot_id, int) and eot_id >= 0:
self.gguf_writer.add_eot_token_id(eot_id)
@@ -152,7 +153,7 @@ class MuseGlimmerAssistantModel(TextModel):
target_arch = json.load(f)["architectures"][0]
target_cls = get_model_class(target_arch)
if target_cls is not type(self):
- target_cls.set_vocab(self) # ty: ignore[unresolved-attribute]
+ target_cls.set_vocab(self) # ty: ignore[missing-attribute]
else:
super().set_vocab()
diff --git a/conversion/muse_spark.py b/conversion/muse_spark.py
index 150f22f95..5378b43ac 100644
--- a/conversion/muse_spark.py
+++ b/conversion/muse_spark.py
@@ -44,7 +44,8 @@ class MuseSparkModel(TextModel):
from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained(self.dir_model)
- eot_id = tok.convert_tokens_to_ids("<|eot|>") # ty: ignore[unresolved-attribute]
+ assert tok is not None
+ eot_id = tok.convert_tokens_to_ids("<|eot|>")
if isinstance(eot_id, int) and eot_id >= 0:
self.gguf_writer.add_eot_token_id(eot_id)
diff --git a/conversion/nemotron.py b/conversion/nemotron.py
index c818a73e1..76aa357ab 100644
--- a/conversion/nemotron.py
+++ b/conversion/nemotron.py
@@ -73,7 +73,7 @@ class LiLiCorrDraftModel(DFlashModel):
try:
if target_arch == "NemotronHForCausalLM":
self.is_moe = "num_experts_per_tok" in target_text_config
- target_cls.set_vocab(self) # ty: ignore[unresolved-attribute]
+ target_cls.set_vocab(self) # ty: ignore[missing-attribute]
finally:
self.dir_model = original_dir
self.gguf_writer.add_mask_token_id(self.hparams["dflash_config"]["mask_token_id"])
@@ -609,6 +609,7 @@ class NemotronHModel(GraniteHybridModel):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+ assert tokenizer is not None
# Pad vocab size (from Mamba2Model/GraniteHybridModel)
self.hparams["pad_vocab_size_multiple"] = 8 # Setting this here since GraniteHybridModel.set_vocab() isn't being invoked now.
@@ -623,14 +624,14 @@ class NemotronHModel(GraniteHybridModel):
vocab_skip = getattr(self, "_vocab_skip", ())
user_defined_tokens = getattr(self, "_user_defined_tokens", None)
- assert max(id_ for tok, id_ in tokenizer.vocab.items() if tok not in vocab_skip) < vocab_size # ty: ignore[unresolved-attribute]
+ assert max(id_ for tok, id_ in tokenizer.vocab.items() if tok not in vocab_skip) < vocab_size
tokpre = self.get_vocab_base_pre(tokenizer)
- reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} # ty: ignore[unresolved-attribute]
- added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
+ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
+ added_vocab = tokenizer.get_added_vocab()
- added_tokens_decoder = tokenizer.added_tokens_decoder # ty: ignore[unresolved-attribute]
+ added_tokens_decoder = tokenizer.added_tokens_decoder
for i in range(vocab_size):
if i not in reverse_vocab:
@@ -641,7 +642,7 @@ class NemotronHModel(GraniteHybridModel):
if token in added_vocab:
if not added_tokens_decoder[i].normalized:
previous_token = token
- token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[unresolved-attribute, invalid-assignment]
+ token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) # ty: ignore[invalid-assignment]
if previous_token != token:
logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
diff --git a/conversion/pockettts.py b/conversion/pockettts.py
index c9198a221..b0cddc685 100644
--- a/conversion/pockettts.py
+++ b/conversion/pockettts.py
@@ -191,7 +191,7 @@ class PocketTTSModel(TextModel):
# unigram segmentation, so use the UGM tokenizer instead
from sentencepiece import sentencepiece_model_pb2 as model
- proto = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+ proto = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
proto.ParseFromString(open(self.dir_model / "tokenizer.model", "rb").read())
assert proto.trainer_spec.model_type == 1, "expected a unigram tokenizer"
diff --git a/conversion/qwen.py b/conversion/qwen.py
index 7c9744f51..83c844ce6 100644
--- a/conversion/qwen.py
+++ b/conversion/qwen.py
@@ -225,12 +225,13 @@ class Qwen3Model(Qwen2Model):
def _find_rerank_config(self):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+ assert tokenizer is not None
self.is_rerank = True
self.is_tied_embeddings = self.hparams.get("tie_word_embeddings", False)
- self.token_false_id = tokenizer.convert_tokens_to_ids("no") # ty: ignore[unresolved-attribute, invalid-assignment]
- self.token_true_id = tokenizer.convert_tokens_to_ids("yes") # ty: ignore[unresolved-attribute, invalid-assignment]
- self.sep_token_id = tokenizer.convert_tokens_to_ids("|") # ty: ignore[unresolved-attribute]
+ self.token_false_id = tokenizer.convert_tokens_to_ids("no") # ty: ignore[invalid-assignment]
+ self.token_true_id = tokenizer.convert_tokens_to_ids("yes") # ty: ignore[invalid-assignment]
+ self.sep_token_id = tokenizer.convert_tokens_to_ids("|")
assert self.token_false_id is not None and self.token_true_id is not None
@@ -329,7 +330,7 @@ class _QwenMtpMixin:
key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None)
type(self)._original_block_count = hparams.get(key)
type(self).opt_num_mtp_layers = 0
- return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[unresolved-attribute]
+ return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[missing-attribute]
@classmethod
def filter_tensors(cls, item):
@@ -366,7 +367,7 @@ class _QwenMtpMixin:
return name, gen
def set_gguf_parameters(self):
- super().set_gguf_parameters() # ty: ignore[unresolved-attribute]
+ super().set_gguf_parameters() # ty: ignore[missing-attribute]
if self.no_mtp:
return
if (n := self.block_count - self.hparams["num_hidden_layers"]) > 0:
@@ -374,15 +375,15 @@ class _QwenMtpMixin:
def prepare_metadata(self, vocab_only: bool):
from_dir = self.fname_out.is_dir()
- super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[unresolved-attribute]
+ super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[missing-attribute]
if not self.mtp_only or not from_dir:
return
- output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+ output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
fname_default: str = gguf.naming_convention(
- self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
- self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+ self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
+ self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
@@ -685,7 +686,7 @@ class _Qwen35MRopeMixin:
rope_parameters: dict
def set_gguf_parameters(self):
- super().set_gguf_parameters() # ty: ignore[unresolved-attribute]
+ super().set_gguf_parameters() # ty: ignore[missing-attribute]
if "mrope_section" not in self.rope_parameters:
self.gguf_writer.add_rope_dimension_sections(self._QWEN35_DEFAULT_MROPE_SECTION)
@@ -843,7 +844,7 @@ class DFlashModel(Qwen3Model):
if target_cls is not type(self):
if target_arch == "NemotronHForCausalLM":
setattr(self, "is_moe", "num_experts_per_tok" in target_hparams)
- target_cls.set_vocab(self) # ty: ignore[unresolved-attribute]
+ target_cls.set_vocab(self) # ty: ignore[missing-attribute]
else:
super().set_vocab()
diff --git a/conversion/t5.py b/conversion/t5.py
index 3466ce49d..e42525794 100644
--- a/conversion/t5.py
+++ b/conversion/t5.py
@@ -40,7 +40,7 @@ class T5Model(TextModel):
if not tokenizer_path.is_file():
raise FileNotFoundError(f"File not found: {tokenizer_path}")
- sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+ sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
# some models like Pile-T5 family use BPE tokenizer instead of Unigram
@@ -178,7 +178,7 @@ class T5EncoderModel(TextModel):
if not tokenizer_path.is_file():
raise FileNotFoundError(f"File not found: {tokenizer_path}")
- sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+ sentencepiece_model = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
# some models like Pile-T5 family use BPE tokenizer instead of Unigram
diff --git a/conversion/xverse.py b/conversion/xverse.py
index aa3b33880..27fcff9d3 100644
--- a/conversion/xverse.py
+++ b/conversion/xverse.py
@@ -25,15 +25,16 @@ class XverseModel(TextModel):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(dir_model)
- vocab_size = hparams.get("vocab_size", len(tokenizer.vocab)) # ty: ignore[unresolved-attribute]
+ assert tokenizer is not None
+ vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))
# Since we are checking the maximum index, we need to ensure it's strictly less than vocab_size,
# because vocab_size is the count of items, and indexes start at 0.
- max_vocab_index = max(tokenizer.get_vocab().values()) # ty: ignore[unresolved-attribute]
+ max_vocab_index = max(tokenizer.get_vocab().values())
if max_vocab_index >= vocab_size:
raise ValueError("Vocabulary size exceeds expected maximum size.")
- reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} # ty: ignore[unresolved-attribute]
- added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
+ reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
+ added_vocab = tokenizer.get_added_vocab()
for token_id in range(vocab_size):
token_text = reverse_vocab[token_id].encode('utf-8')
@@ -44,7 +45,7 @@ class XverseModel(TextModel):
elif re.fullmatch(br"<0x[0-9A-Fa-f]{2}>", token_text):
toktype = gguf.TokenType.BYTE # special
elif reverse_vocab[token_id] in added_vocab:
- if tokenizer.added_tokens_decoder[token_id].special: # ty: ignore[unresolved-attribute]
+ if tokenizer.added_tokens_decoder[token_id].special:
toktype = gguf.TokenType.CONTROL
else:
toktype = gguf.TokenType.USER_DEFINED
diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py
index 3b8a33597..4340a9fee 100755
--- a/convert_hf_to_gguf_update.py
+++ b/convert_hf_to_gguf_update.py
@@ -325,7 +325,8 @@ for model in [*pre_computed_hashes, *all_models]:
except Exception as e:
raise OSError(f"Error loading tokenizer for model {name}.") from e
- chktok = tokenizer.encode(CHK_TXT) # ty: ignore[unresolved-attribute]
+ assert tokenizer is not None
+ chktok = tokenizer.encode(CHK_TXT)
chkhsh = sha256(str(chktok).encode()).hexdigest()
logger.info(f"model: {name}")
@@ -486,6 +487,7 @@ for model in models:
logger.error(f"Failed to load tokenizer for model {name}. Error: {e}")
continue # Skip this model and continue with the next one in the loop
+ assert tokenizer is not None
if not os.path.exists(f"models/ggml-vocab-{name}.gguf"):
logger.info(f"Skip vocab files for model {name}, no GGUF file found")
continue
@@ -497,7 +499,7 @@ for model in models:
with open(f"models/ggml-vocab-{name}.gguf.out", "w") as f:
for text in tests:
- res = tokenizer.encode(text, add_special_tokens=False) # ty: ignore[unresolved-attribute]
+ res = tokenizer.encode(text, add_special_tokens=False)
for r in res:
f.write(f" {r}")
f.write("\n")
diff --git a/convert_lora_to_gguf.py b/convert_lora_to_gguf.py
index f9adecbb1..a6c9002aa 100755
--- a/convert_lora_to_gguf.py
+++ b/convert_lora_to_gguf.py
@@ -443,7 +443,8 @@ if __name__ == '__main__':
# the invocation string includes the "<|start_of_turn|>"
# token, but the adapters themselves were trained to
# activate _after_ that first token, so we drop it here.
- alora_invocation_tokens = tokenizer(invocation_string)["input_ids"][1:] # ty: ignore[call-non-callable]
+ assert tokenizer is not None
+ alora_invocation_tokens = tokenizer(invocation_string)["input_ids"][1:]
if alora_invocation_tokens:
logger.debug("GGUF KV: %s = %s", gguf.Keys.Adapter.ALORA_INVOCATION_TOKENS, alora_invocation_tokens)
self.gguf_writer.add_key_value(
diff --git a/examples/model-conversion/scripts/causal/compare-logits.py b/examples/model-conversion/scripts/causal/compare-logits.py
index 181c04863..96dd78bc2 100755
--- a/examples/model-conversion/scripts/causal/compare-logits.py
+++ b/examples/model-conversion/scripts/causal/compare-logits.py
@@ -7,7 +7,7 @@ import os
# Add utils directory to path for direct script execution
sys.path.insert(0, str(Path(__file__).parent.parent / "utils"))
-from common import get_model_name_from_env_path, compare_tokens, exit_with_warning # type: ignore[import-not-found, ty:unresolved-import]
+from common import get_model_name_from_env_path, compare_tokens, exit_with_warning # ty: ignore[import-not-found]
def quick_logits_check(pytorch_file, llamacpp_file):
"""Lightweight sanity check before NMSE"""
diff --git a/examples/model-conversion/scripts/causal/run-casual-gen-embeddings-org.py b/examples/model-conversion/scripts/causal/run-casual-gen-embeddings-org.py
index cb840dd55..31ac0f3dd 100755
--- a/examples/model-conversion/scripts/causal/run-casual-gen-embeddings-org.py
+++ b/examples/model-conversion/scripts/causal/run-casual-gen-embeddings-org.py
@@ -33,6 +33,7 @@ print("EOS token id: ", config.eos_token_id)
print("Loading model and tokenizer using AutoTokenizer:", model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)
+assert tokenizer is not None
if unreleased_model_name:
model_name_lower = unreleased_model_name.lower()
@@ -56,11 +57,11 @@ model_name = os.path.basename(model_path)
print(f"Model name: {model_name}")
prompt = "Hello world today"
-input_ids = tokenizer(prompt, return_tensors="pt").input_ids # ty: ignore[call-non-callable]
+input_ids = tokenizer(prompt, return_tensors="pt").input_ids
token_ids = input_ids[0].cpu().tolist()
print(f"Input tokens: {input_ids}")
print(f"Input text: {repr(prompt)}")
-print(f"Tokenized: {tokenizer.convert_ids_to_tokens(input_ids[0])}") # ty: ignore[unresolved-attribute]
+print(f"Tokenized: {tokenizer.convert_ids_to_tokens(input_ids[0])}")
with torch.no_grad():
outputs = model(input_ids, output_hidden_states=True)
@@ -83,7 +84,7 @@ with torch.no_grad():
# Print embeddings per token in the requested format
print("\nToken embeddings:")
- tokens = tokenizer.convert_ids_to_tokens(input_ids[0]) # ty: ignore[unresolved-attribute]
+ tokens = tokenizer.convert_ids_to_tokens(input_ids[0])
for i, embedding in enumerate(token_embeddings):
# Format: show first few values, ..., then last few values
if len(embedding) > 10:
diff --git a/examples/model-conversion/scripts/embedding/run-original-model.py b/examples/model-conversion/scripts/embedding/run-original-model.py
index 001d58896..da6aec773 100755
--- a/examples/model-conversion/scripts/embedding/run-original-model.py
+++ b/examples/model-conversion/scripts/embedding/run-original-model.py
@@ -64,7 +64,7 @@ def load_model_and_tokenizer(model_path, use_sentence_transformers=False, device
print("Using SentenceTransformer to apply all numbered layers")
model = SentenceTransformer(model_path)
tokenizer = model.tokenizer
- config = model[0].auto_model.config # ty: ignore[unresolved-attribute]
+ config = model[0].auto_model.config # ty: ignore[missing-union-attribute]
else:
tokenizer = AutoTokenizer.from_pretrained(model_path)
config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
diff --git a/examples/model-conversion/scripts/utils/check-nmse.py b/examples/model-conversion/scripts/utils/check-nmse.py
index 324e3858e..363d80326 100755
--- a/examples/model-conversion/scripts/utils/check-nmse.py
+++ b/examples/model-conversion/scripts/utils/check-nmse.py
@@ -5,7 +5,7 @@ import sys
import os
import argparse
from pathlib import Path
-from common import get_model_name_from_env_path # type: ignore[import-not-found, ty:unresolved-import]
+from common import get_model_name_from_env_path # ty: ignore[import-not-found]
def calculate_nmse(reference, test):
mse = np.mean((test - reference) ** 2)
diff --git a/examples/model-conversion/scripts/utils/compare_tokens.py b/examples/model-conversion/scripts/utils/compare_tokens.py
index 55e3f26ab..0d4612ef8 100755
--- a/examples/model-conversion/scripts/utils/compare_tokens.py
+++ b/examples/model-conversion/scripts/utils/compare_tokens.py
@@ -2,7 +2,7 @@
import argparse
import sys
-from common import compare_tokens # type: ignore[import-not-found, ty:unresolved-import]
+from common import compare_tokens # ty: ignore[import-not-found]
def parse_arguments():
diff --git a/examples/model-conversion/scripts/utils/semantic_check.py b/examples/model-conversion/scripts/utils/semantic_check.py
index 754ae733d..f600bcade 100644
--- a/examples/model-conversion/scripts/utils/semantic_check.py
+++ b/examples/model-conversion/scripts/utils/semantic_check.py
@@ -7,7 +7,7 @@ import importlib
from pathlib import Path
from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM, AutoModel
-from common import compare_tokens, exit_with_warning # type: ignore[import-not-found, ty:unresolved-import]
+from common import compare_tokens, exit_with_warning # ty: ignore[import-not-found]
unreleased_model_name = os.getenv('UNRELEASED_MODEL_NAME')
@@ -183,6 +183,7 @@ def main():
# Load the python model to get configuration information and also to load the tokenizer.
print("Loading model and tokenizer using AutoTokenizer:", args.model_path)
tokenizer = AutoTokenizer.from_pretrained(args.model_path)
+ assert tokenizer is not None
config = AutoConfig.from_pretrained(args.model_path, trust_remote_code=True)
if unreleased_model_name:
@@ -207,8 +208,8 @@ def main():
else:
model = AutoModel.from_pretrained(args.model_path, trust_remote_code=True)
- encoded = tokenizer(prompt, return_tensors="pt") # ty: ignore[call-non-callable]
- tokens = tokenizer.convert_ids_to_tokens(encoded['input_ids'][0]) # ty: ignore[unresolved-attribute]
+ encoded = tokenizer(prompt, return_tensors="pt")
+ tokens = tokenizer.convert_ids_to_tokens(encoded['input_ids'][0])
n_tokens = len(tokens)
print(f"n_tokens: {n_tokens}");
print(f"hidden_size: {model.config.hidden_size}")
diff --git a/examples/pydantic_models_to_grammar.py b/examples/pydantic_models_to_grammar.py
index 736b2b7df..ff6e120e4 100644
--- a/examples/pydantic_models_to_grammar.py
+++ b/examples/pydantic_models_to_grammar.py
@@ -1285,7 +1285,7 @@ def convert_dictionary_to_pydantic_model(dictionary: dict[str, Any], model_name:
if items != {}:
array = {"properties": items}
array_type = convert_dictionary_to_pydantic_model(array, f"{model_name}_{field_name}_items")
- fields[field_name] = (list[array_type], ...) # ty: ignore[invalid-type-form]
+ fields[field_name] = (list[array_type], ...)
else:
fields[field_name] = (list, ...)
elif field_type == "object":
diff --git a/gguf-py/gguf/lazy.py b/gguf-py/gguf/lazy.py
index a39f22321..eae6ab5c4 100644
--- a/gguf-py/gguf/lazy.py
+++ b/gguf-py/gguf/lazy.py
@@ -138,7 +138,7 @@ class LazyBase(ABC, metaclass=LazyMeta):
if isinstance(meta_noop, tuple):
dtype, shape = meta_noop
assert callable(shape)
- res = cls.meta_with_dtype_and_shape(dtype, shape(res.shape)) # ty: ignore[call-top-callable]
+ res = cls.meta_with_dtype_and_shape(dtype, shape(res.shape))
else:
res = cls.meta_with_dtype_and_shape(meta_noop, res.shape)
diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py
index e49a357df..9b8b87cf6 100644
--- a/gguf-py/gguf/vocab.py
+++ b/gguf-py/gguf/vocab.py
@@ -14,12 +14,12 @@ except ImportError:
SentencePieceProcessor: Any = None
try:
- from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # type: ignore[import-not-found, ty:unresolved-import]
- from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import]
- from mistral_common.tokens.tokenizers.utils import ( # type: ignore[import-not-found, ty:unresolved-import]
+ from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # ty: ignore[import-not-found]
+ from mistral_common.tokens.tokenizers.tekken import Tekkenizer # ty: ignore[import-not-found]
+ from mistral_common.tokens.tokenizers.utils import ( # ty: ignore[import-not-found]
_filter_valid_tokenizer_files,
)
- from mistral_common.tokens.tokenizers.sentencepiece import ( # type: ignore[import-not-found, ty:unresolved-import]
+ from mistral_common.tokens.tokenizers.sentencepiece import ( # ty: ignore[import-not-found]
SentencePieceTokenizer,
)
except ImportError:
@@ -32,7 +32,7 @@ else:
_mistral_common_installed = True
try:
- from mistral_common.tokens.tokenizers.utils import ( # type: ignore[import-not-found, ty:unresolved-import]
+ from mistral_common.tokens.tokenizers.utils import ( # ty: ignore[import-not-found]
get_one_valid_tokenizer_file,
)
except ImportError:
@@ -629,7 +629,8 @@ class LlamaHfVocab(Vocab):
cache_dir=base_path,
local_files_only=True,
)
- assert self.tokenizer.is_fast # assume tokenizer.json is used # ty: ignore[unresolved-attribute]
+ assert self.tokenizer is not None
+ assert self.tokenizer.is_fast # assume tokenizer.json is used
# Initialize lists and dictionaries for added tokens
self.added_tokens_list = []
@@ -638,30 +639,31 @@ class LlamaHfVocab(Vocab):
# Process added tokens
for tok, tokidx in sorted(
- self.tokenizer.get_added_vocab().items(), key=lambda x: x[1] # ty: ignore[unresolved-attribute]
+ self.tokenizer.get_added_vocab().items(), key=lambda x: x[1]
):
# Only consider added tokens that are not in the base vocabulary
- if tokidx >= self.tokenizer.vocab_size: # ty: ignore[unresolved-attribute]
+ if tokidx >= self.tokenizer.vocab_size:
self.added_tokens_list.append(tok)
self.added_tokens_dict[tok] = tokidx
self.added_tokens_ids.add(tokidx)
# Store special tokens and their IDs
self.specials = {
- tok: self.tokenizer.get_vocab()[tok] # ty: ignore[unresolved-attribute]
- for tok in self.tokenizer.all_special_tokens # ty: ignore[unresolved-attribute]
+ tok: self.tokenizer.get_vocab()[tok]
+ for tok in self.tokenizer.all_special_tokens
}
- self.special_ids = set(self.tokenizer.all_special_ids) # ty: ignore[unresolved-attribute]
+ self.special_ids = set(self.tokenizer.all_special_ids)
# Set vocabulary sizes
- self.vocab_size_base = self.tokenizer.vocab_size # ty: ignore[unresolved-attribute]
+ self.vocab_size_base = self.tokenizer.vocab_size
self.vocab_size = self.vocab_size_base + len(self.added_tokens_list)
self.fname_tokenizer = fname_tokenizer
def hf_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
+ assert self.tokenizer is not None
reverse_vocab = {
- id: encoded_tok for encoded_tok, id in self.tokenizer.get_vocab().items() # ty: ignore[unresolved-attribute]
+ id: encoded_tok for encoded_tok, id in self.tokenizer.get_vocab().items()
}
for token_id in range(self.vocab_size_base):
@@ -702,7 +704,8 @@ class LlamaHfVocab(Vocab):
yield text.encode("utf-8"), score, toktype
def has_newline_token(self):
- return "<0x0A>" in self.tokenizer.vocab or "\n" in self.tokenizer.vocab # ty: ignore[unresolved-attribute]
+ assert self.tokenizer is not None
+ return "<0x0A>" in self.tokenizer.vocab or "\n" in self.tokenizer.vocab
def all_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
yield from self.hf_tokens()
diff --git a/scripts/jinja/jinja-tester.py b/scripts/jinja/jinja-tester.py
index 6d36ecfa5..f37350ec1 100755
--- a/scripts/jinja/jinja-tester.py
+++ b/scripts/jinja/jinja-tester.py
@@ -241,10 +241,10 @@ class CodeEditor(QPlainTextEdit):
if not self.isReadOnly():
selection = QTextEdit.ExtraSelection()
line_color = QColorConstants.Yellow.lighter(160)
- selection.format.setBackground(line_color) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
- selection.format.setProperty(QTextFormat.Property.FullWidthSelection, True) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
- selection.cursor = self.textCursor() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
- selection.cursor.clearSelection() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+ selection.format.setBackground(line_color) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
+ selection.format.setProperty(QTextFormat.Property.FullWidthSelection, True) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
+ selection.cursor = self.textCursor() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
+ selection.cursor.clearSelection() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
extra_selections.append(selection)
self.setExtraSelections(extra_selections)
@@ -262,8 +262,8 @@ class CodeEditor(QPlainTextEdit):
)
extra = QTextEdit.ExtraSelection()
- extra.format.setBackground(color.lighter(160)) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
- extra.cursor = cursor # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+ extra.format.setBackground(color.lighter(160)) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
+ extra.cursor = cursor # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
self.setExtraSelections(self.extraSelections() + [extra])
@@ -274,8 +274,8 @@ class CodeEditor(QPlainTextEdit):
cursor.select(QTextCursor.SelectionType.LineUnderCursor)
extra = QTextEdit.ExtraSelection()
- extra.format.setBackground(color.lighter(160)) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
- extra.cursor = cursor # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+ extra.format.setBackground(color.lighter(160)) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
+ extra.cursor = cursor # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
self.setExtraSelections(self.extraSelections() + [extra])
diff --git a/scripts/tool_bench.py b/scripts/tool_bench.py
index fb7df10f3..2015f6682 100755
--- a/scripts/tool_bench.py
+++ b/scripts/tool_bench.py
@@ -52,8 +52,8 @@ import typer
sys.path.insert(0, Path(__file__).parent.parent.as_posix())
if True:
- from tools.server.tests.utils import ServerProcess # ty: ignore[unresolved-import]
- from tools.server.tests.unit.test_tool_call import do_test_calc_result, do_test_hello_world, do_test_weather # ty: ignore[unresolved-import]
+ from tools.server.tests.utils import ServerProcess # ty: ignore[import-not-found]
+ from tools.server.tests.unit.test_tool_call import do_test_calc_result, do_test_hello_world, do_test_weather # ty: ignore[import-not-found]
@contextmanager
diff --git a/tests/test-tokenizer-0.py b/tests/test-tokenizer-0.py
index 4f3f1c8a6..334b2df96 100644
--- a/tests/test-tokenizer-0.py
+++ b/tests/test-tokenizer-0.py
@@ -12,6 +12,7 @@ dir_tokenizer = args.dir_tokenizer
fname_tok = args.fname_tok
tokenizer = AutoTokenizer.from_pretrained(dir_tokenizer)
+assert tokenizer is not None
print('tokenizing file: ', fname_tok) # noqa: NP100
fname_out = fname_tok + '.tok'
@@ -19,7 +20,7 @@ with open(fname_tok, 'r', encoding='utf-8') as f:
lines = f.readlines()
s = ''.join(lines)
t_start = time.time()
- res = tokenizer.encode(s, add_special_tokens=False) # ty: ignore[unresolved-attribute]
+ res = tokenizer.encode(s, add_special_tokens=False)
t_end = time.time()
print('\nmain : tokenized in', "{:.3f}".format(1000.0 * (t_end - t_start)), 'ms (py)') # noqa: NP100
with open(fname_out, 'w', encoding='utf-8') as f:
diff --git a/ty.toml b/ty.toml
index 340b0649d..1a11151e6 100644
--- a/ty.toml
+++ b/ty.toml
@@ -19,8 +19,9 @@ include = [
[overrides.rules]
unresolved-reference = "ignore"
-unresolved-import = "ignore"
-unresolved-attribute = "ignore"
+import-not-found = "ignore"
+missing-attribute = "ignore"
+missing-union-attribute = "ignore"
[[overrides]]
include = [
@@ -30,3 +31,4 @@ include = [
[overrides.rules]
unsupported-operator = "ignore"
not-subscriptable = "ignore"
+invalid-type-form = "ignore"