Commit 7d6fd845e for llama.cpp

commit 7d6fd845ecbb22fa941848f493bd81a971afb6ec
Author: Ramzi <146260554+rmz-24@users.noreply.github.com>
Date:   Tue Nov 10 13:20:56 2026 +0100

    convert : mark Sarvam byte fallback tokens as BYTE (#32006)

diff --git a/conversion/bailingmoe.py b/conversion/bailingmoe.py
index 351be1df1..18fe8a2b7 100644
--- a/conversion/bailingmoe.py
+++ b/conversion/bailingmoe.py
@@ -1,5 +1,6 @@
 from __future__ import annotations

+import re
 from typing import Callable, Iterable, TYPE_CHECKING

 import torch
@@ -198,6 +199,18 @@ class SarvamMoEModel(BailingMoeV2Model):
     #  - full rotary (no partial_rotary_factor)
     #  - expert bias is zero-mean normalized at load time

+    def set_vocab(self):
+        # BPE with byte fallback: mark <0xXX> tokens as BYTE so they decode to raw bytes
+        tokens, toktypes, tokpre = self.get_vocab_base()
+        toktypes = [gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-F]{2}>", tok) else typ for tok, typ in zip(tokens, toktypes)]
+        self.gguf_writer.add_tokenizer_model("gpt2")
+        self.gguf_writer.add_tokenizer_pre(tokpre)
+        self.gguf_writer.add_token_list(tokens)
+        self.gguf_writer.add_token_types(toktypes)
+
+        special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+        special_vocab.add_to_gguf(self.gguf_writer)
+
     def set_gguf_parameters(self):
         super().set_gguf_parameters()
         hparams = self.hparams