Commit 351025ec5 for llama.cpp
commit 351025ec532f825c7eb6c7ae1017d8e6d46d0cc9
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Tue Nov 10 11:28:47 2026 +0100
mtmd: add kimi K3.1 vision (#31980)
* mtmd: support kimi k3.1 vision
* fix preproc (both k2.5 and k3.1) and projector mlp
* small renaming
diff --git a/conversion/__init__.py b/conversion/__init__.py
index f9dabf4f2..6800093ae 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -371,6 +371,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"InternVisionModel": "internvl",
"JanusForConditionalGeneration": "januspro",
"KimiK25ForConditionalGeneration": "kimivl",
+ "KimiK31ForConditionalGeneration": "kimivl",
"KimiVLForConditionalGeneration": "kimivl",
"Lfm2AudioForConditionalGeneration": "lfm2",
"D1OmniModel": "lfm2",
diff --git a/conversion/kimivl.py b/conversion/kimivl.py
index ae60abf30..8234ac027 100644
--- a/conversion/kimivl.py
+++ b/conversion/kimivl.py
@@ -57,6 +57,8 @@ class KimiVLModel(MmprojModel):
class KimiK25Model(MmprojModel):
"""Kimi-K2.5 with MoonViT3d vision encoder"""
+ projector_type = gguf.VisionProjectorType.KIMIK25
+
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -78,7 +80,7 @@ class KimiK25Model(MmprojModel):
super().set_gguf_parameters()
assert self.hparams_vision is not None
- self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.KIMIK25)
+ self.gguf_writer.add_clip_projector_type(self.projector_type)
# Position embedding parameters (for interpolation)
self.gguf_writer.add_uint32("vision.pos_emb_height", self.hparams_vision.get("init_pos_emb_height", 64))
@@ -87,7 +89,11 @@ class KimiK25Model(MmprojModel):
# Projector parameters
self.gguf_writer.add_vision_use_gelu(self.hparams_vision.get("projector_hidden_act", "gelu") == "gelu")
- self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("projector_ln_eps", 1e-5))
+ if self.hparams_vision.get("norm_type", "layernorm") == "rmsnorm":
+ # nn.RMSNorm without eps uses finfo(float32).eps, also for bf16 inputs
+ self.gguf_writer.add_vision_attention_layernorm_eps(torch.finfo(torch.float32).eps)
+ else:
+ self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("projector_ln_eps", 1e-5))
self.gguf_writer.add_vision_projector_scale_factor(self.merge_kernel_size[0])
# Image size limits
@@ -120,7 +126,7 @@ class KimiK25Model(MmprojModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
assert self.hparams_vision is not None
- n_head = self.hparams_vision.get("num_attention_heads", 16)
+ n_head = self.find_vparam(["num_attention_heads", "vt_num_attention_heads"], optional=True) or 16
# Permute Q/K weights/biases from interleaved to split RoPE format
# This allows using build_rope_2d at runtime without post-permutation.
@@ -171,3 +177,19 @@ class Glm5vModel(KimiK25Model):
name = name.replace("mm_projector.linear_", "mm_projector.proj.linear_", 1)
yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("KimiK31ForConditionalGeneration")
+@ModelBase.example("moonshotai/Kimi-K3.1-Pro")
+class KimiK31VisionModel(KimiK25Model):
+ """Kimi-K2.5 MoonViT3d with RMSNorm, a wider QKV and the patchmergerv2 projector"""
+
+ projector_type = gguf.VisionProjectorType.KIMIK31
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ assert self.hparams_vision is not None
+ assert self.hparams_vision["norm_type"] == "rmsnorm"
+ assert self.hparams_vision["mm_projector_type"] == "patchmergerv2"
+ n_head = self.find_vparam(["vt_num_attention_heads"])
+ self.gguf_writer.add_vision_head_dim(self.hparams_vision["qkv_hidden_size"] // n_head)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 5f9cbf1e3..30185f02c 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -7044,6 +7044,7 @@ class VisionProjectorType:
KIMIVL = "kimivl"
PADDLEOCR = "paddleocr"
KIMIK25 = "kimik25"
+ KIMIK31 = "kimik31"
LIGHTONOCR = "lightonocr"
COGVLM = "cogvlm"
JANUS_PRO = "janus_pro"
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index 8888dc76e..bb7d6dcae 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -2282,6 +2282,7 @@ class TensorNameMap:
"vision_tower.post_trunk_norm", # dots.ocr
"vision_encoder.post_trunk_norm", # dots3note
"vit.perceive.after_rms", # HunyuanVL
+ "mm_projector.post_norm", # Kimi-K3.1
),
MODEL_TENSOR.V_MM_INP_PROJ: (
diff --git a/tools/mtmd/models/kimik25.cpp b/tools/mtmd/models/kimik25.cpp
index 13baf86d0..b77ea9604 100644
--- a/tools/mtmd/models/kimik25.cpp
+++ b/tools/mtmd/models/kimik25.cpp
@@ -40,7 +40,9 @@ ggml_cgraph * mtmd_graph_kimik25::build() {
ggml_set_name(pos_w, "pos_w");
ggml_set_input(pos_w);
- ggml_tensor * learned_pos_embd = resize_position_embeddings_3d(GGML_SCALE_MODE_BICUBIC);
+ const bool is_k31 = proj_type == PROJECTOR_TYPE_KIMIK31;
+
+ ggml_tensor * learned_pos_embd = resize_position_embeddings_3d(is_k31 ? GGML_SCALE_MODE_BILINEAR : GGML_SCALE_MODE_BICUBIC);
// Kimi-K2.5 uses interleaved 2D RoPE pattern natively, but
// Q / K are permuted during conversion to use split format.
@@ -57,14 +59,25 @@ ggml_cgraph * mtmd_graph_kimik25::build() {
ggml_tensor * cur = build_vit(
inp, n_patches,
- NORM_TYPE_NORMAL,
+ is_k31 ? NORM_TYPE_RMS : NORM_TYPE_NORMAL,
hparams.ffn_op,
nullptr,
add_pos);
cb(cur, "vit_out", -1);
- {
+ if (is_k31) {
+ // patchmergerv2: no pre-norm, RMS norm after the MLP
+ cur = build_patch_merge_permute(cur, hparams.n_merge);
+ cur = build_ffn(cur,
+ model.mm_1_w, nullptr,
+ nullptr, nullptr,
+ model.mm_2_w, nullptr,
+ FFN_GELU_ERF,
+ -1);
+ cur = build_norm(cur, model.mm_post_norm_w, nullptr, NORM_TYPE_RMS, 1e-5f, -1);
+ cb(cur, "proj_out", -1);
+ } else {
// patch_merger
const int scale_factor = model.hparams.n_merge;
cur = build_patch_merge_permute(cur, scale_factor);
@@ -88,7 +101,7 @@ ggml_cgraph * mtmd_graph_kimik25::build() {
model.mm_1_w, model.mm_1_b,
nullptr, nullptr,
model.mm_2_w, model.mm_2_b,
- FFN_GELU,
+ FFN_GELU_ERF,
-1);
cb(cur, "proj_out", -1);
diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp
index 3242bddbb..738837152 100644
--- a/tools/mtmd/mtmd-graph.cpp
+++ b/tools/mtmd/mtmd-graph.cpp
@@ -885,6 +885,7 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
builder = std::make_unique<mtmd_graph_paddleocr>(ctx, img);
} break;
case PROJECTOR_TYPE_KIMIK25:
+ case PROJECTOR_TYPE_KIMIK31:
{
builder = std::make_unique<mtmd_graph_kimik25>(ctx, img);
} break;
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index b0543aa43..33ce243d5 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -818,6 +818,45 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const mtmd_i
return output;
}
+//
+// mtmd_image_preprocessor_kimik25
+//
+
+mtmd_image_preproc_out mtmd_image_preprocessor_kimik25::preprocess(const mtmd_image_u8 & img) const {
+ GGML_ASSERT(hparams.image_max_pixels > 0);
+ // patch_limit_on_one_side from the preprocessor config, same for K2.5 and K3.1
+ const int side_limit = 512;
+ const int ps = hparams.patch_size;
+ const int64_t in_patch_limit = hparams.image_max_pixels / (ps * ps);
+ const int width = img.get_size().width;
+ const int height = img.get_size().height;
+
+ const double s1 = std::sqrt((double) in_patch_limit
+ / (std::max(1.0, (double) (width / ps)) * std::max(1.0, (double) (height / ps))));
+ const double s2 = (double) side_limit * ps / width;
+ const double s3 = (double) side_limit * ps / height;
+ const double scale = std::min({1.0, s1, s2, s3});
+ const int new_w = std::min(std::max(1, (int) (width * scale)), side_limit * ps);
+ const int new_h = std::min(std::max(1, (int) (height * scale)), side_limit * ps);
+
+ mtmd_image_u8 content;
+ if (new_w == width && new_h == height) {
+ content = img;
+ } else {
+ img_tool::resize(img, content, mtmd_image_size{new_w, new_h}, hparams.image_resize_algo, PAD_NONE);
+ }
+
+ const int align = ps * hparams.n_merge;
+ mtmd_image_u8 canvas;
+ canvas.set_size(mtmd_image_size{MTMD_ALIGN(new_w, align), MTMD_ALIGN(new_h, align)}, img.is_placeholder());
+ img_tool::fill(canvas, {0, 0, 0});
+ img_tool::composite(canvas, content, 0, 0);
+
+ mtmd_image_preproc_out output;
+ output.append(hparams, canvas, true);
+ return output;
+}
+
//
// mtmd_image_preprocessor_qwen3vl
//
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index 2cb9454f0..3e0b127e9 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -143,6 +143,12 @@ struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {
virtual mtmd_image_size target_size(const mtmd_image_size & original_size) const { return get_target_size(hparams, original_size); }
};
+// Kimi-K2.5 / K3.1 navit_resize_image: scale down to the patch limits, then pad bottom-right with black to a multiple of patch_size*n_merge
+struct mtmd_image_preprocessor_kimik25 : mtmd_image_preprocessor {
+ mtmd_image_preprocessor_kimik25(const mtmd_model * model) : mtmd_image_preprocessor(model) {}
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
+};
+
// Qwen3-VL, same as dyn_size for images, video frames are sized as Qwen3VLVideoProcessor
struct mtmd_image_preprocessor_qwen3vl : mtmd_image_preprocessor_dyn_size {
using mtmd_image_preprocessor_dyn_size::mtmd_image_preprocessor_dyn_size;
diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h
index e245f4914..61293aadb 100644
--- a/tools/mtmd/mtmd-impl.h
+++ b/tools/mtmd/mtmd-impl.h
@@ -548,6 +548,7 @@ enum projector_type {
PROJECTOR_TYPE_YOUTUVL,
PROJECTOR_TYPE_YASA2,
PROJECTOR_TYPE_KIMIK25,
+ PROJECTOR_TYPE_KIMIK31,
PROJECTOR_TYPE_NEMOTRON_V2_VL,
PROJECTOR_TYPE_NEMOTRON_OMNI,
PROJECTOR_TYPE_HUNYUANVL,
@@ -634,6 +635,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_YOUTUVL, "youtuvl"},
{ PROJECTOR_TYPE_YASA2, "yasa2"},
{ PROJECTOR_TYPE_KIMIK25, "kimik25"},
+ { PROJECTOR_TYPE_KIMIK31, "kimik31"},
{ PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"},
{ PROJECTOR_TYPE_NEMOTRON_OMNI, "nemotron_omni"},
{ PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"},
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index ae7a942ac..e299f0896 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -546,6 +546,7 @@ struct mtmd_model_loader {
hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_KIMIK25:
+ case PROJECTOR_TYPE_KIMIK31:
{
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.rope_theta = 10000.0f;
@@ -561,6 +562,9 @@ struct mtmd_model_loader {
} else {
hparams.set_limit_image_tokens(2, 4096);
}
+ if (model.proj_type == PROJECTOR_TYPE_KIMIK31) {
+ hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
+ }
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
@@ -2143,6 +2147,12 @@ struct mtmd_model_loader {
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
+ case PROJECTOR_TYPE_KIMIK31:
+ {
+ model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
+ model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
+ model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
+ } break;
case PROJECTOR_TYPE_DEEPSEEK4V:
{
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
@@ -3937,6 +3947,7 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
case PROJECTOR_TYPE_D1OMNI_V:
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_KIMIK25:
+ case PROJECTOR_TYPE_KIMIK31:
{
// dynamic size
int out_patch_size = params.patch_size * hparams.n_merge;
@@ -4278,6 +4289,7 @@ int mtmd_model::n_mmproj_embd() const {
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_KIMIK25:
+ case PROJECTOR_TYPE_KIMIK31:
case PROJECTOR_TYPE_YASA2:
case PROJECTOR_TYPE_DEEPSEEK4V:
case PROJECTOR_TYPE_DEEPSEEK41V:
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index 4c2b15107..768eab562 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -1362,6 +1362,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_KIMIK25:
+ case PROJECTOR_TYPE_KIMIK31:
case PROJECTOR_TYPE_LIGHTONOCR:
{
// set the 2D positions
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 1399b7ad4..aa886b6c7 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -635,7 +635,10 @@ struct mtmd_context {
bool ov_img_first = false;
// MiniCPM-V 4.6/4.7 prepends an <image_id>N</image_id> tag before <image>
- bool use_image_id = false;
+ bool has_prompt_img_id = false;
+
+ // Kimi-K3.1 writes the original image size after img_beg
+ bool has_prompt_img_size = false;
// string template for slice image delimiters with row/col (idefics3)
std::string sli_img_start_tmpl;
@@ -830,7 +833,7 @@ struct mtmd_context {
tok_row_end = {lookup_token("\n")};
tok_row_end_trail = false; // no trailing end-of-row token
ov_img_first = true;
- use_image_id = true;
+ has_prompt_img_id = true;
if (ctx_v->model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0) {
image_preproc = std::make_unique<mtmd_image_preprocessor_minicpm_ocr_1_0>(models.v.get());
} else {
@@ -995,12 +998,21 @@ struct mtmd_context {
// <|begin_of_image|> ... (image embeddings) ... <|end_of_image|>
img_beg = "<|begin_of_image|>";
img_end = "<|end_of_image|>";
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} else {
// <|media_begin|> ... (image embeddings) ... <|media_end|>
img_beg = "<|media_begin|>";
img_end = "<|media_end|>";
+ image_preproc = std::make_unique<mtmd_image_preprocessor_kimik25>(models.v.get());
}
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
+ } break;
+ case PROJECTOR_TYPE_KIMIK31:
+ {
+ // <|media_begin|>image WxH<|media_content|> ... (image embeddings) ... <|media_end|>
+ img_beg = "<|media_begin|>";
+ img_end = "<|media_end|>";
+ has_prompt_img_size = true;
+ image_preproc = std::make_unique<mtmd_image_preprocessor_kimik25>(models.v.get());
} break;
case PROJECTOR_TYPE_LIGHTONOCR:
{
@@ -1622,6 +1634,9 @@ struct mtmd_tokenizer {
if (!ctx->img_beg.empty()) {
add_text(ctx->img_beg, true); // add image begin token
}
+ if (ctx->has_prompt_img_size) {
+ add_text(string_format("image %ux%u<|media_content|>", bitmaps[0]->nx, bitmaps[0]->ny), true);
+ }
// TODO @ngxson : this is quite hacky because preprocessor only support batch with one single element, that need to be fixed in the future (e.g. by changing the preprocessor interface always take single input)
@@ -1676,7 +1691,7 @@ struct mtmd_tokenizer {
if (has_tiling_grid && ctx->proj_type_v() == PROJECTOR_TYPE_MINICPMV4_7) {
GGML_ASSERT(bitmaps.size() == 1);
- if (ctx->use_image_id) {
+ if (ctx->has_prompt_img_id) {
add_text("<image_id>" + std::to_string(n_images_added) + "</image_id>", true);
}
add_text(ctx->tok_ov_img_start);
@@ -1701,7 +1716,7 @@ struct mtmd_tokenizer {
// add overview image (first)
if (ctx->ov_img_first) {
- if (ctx->use_image_id) {
+ if (ctx->has_prompt_img_id) {
add_text("<image_id>" + std::to_string(n_images_added) + "</image_id>", true);
}
add_text(ctx->tok_ov_img_start);