Commit a08ab75ae for llama.cpp
commit a08ab75ae74c36c15accc6c682a989a9e98dbfd2
Author: tc-mb <157115220+tc-mb@users.noreply.github.com>
Date: Fri Oct 23 01:22:15 2026 +0800
mtmd : add MiniCPM-OCR 1.0 support (#30979)
Signed-off-by: tc-mb <tianchi_cai@icloud.com>
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 8921f53e4..5f252b490 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -196,6 +196,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"MiniCPMForCausalLM": "minicpm",
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
"MiniCPMV4_7ForConditionalGeneration": "minicpm",
+ "MiniCPMOcrForConditionalGeneration": "minicpm",
"MiniMaxText01ForCausalLM": "minimax",
"MiniMaxM1ForCausalLM": "minimax",
"MiniMaxM2ForCausalLM": "minimax",
@@ -361,6 +362,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"MiniMaxM3SparseForConditionalGeneration": "minimax",
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
"MiniCPMV4_7ForConditionalGeneration": "minicpm",
+ "MiniCPMOcrForConditionalGeneration": "minicpm",
"Mistral3ForConditionalGeneration": "llava",
"Mistral3Model": "llava",
"NemotronH_Omni_Reasoning_V3": "nemotron",
diff --git a/conversion/minicpm.py b/conversion/minicpm.py
index 937ac033a..d8fceb0c0 100644
--- a/conversion/minicpm.py
+++ b/conversion/minicpm.py
@@ -272,3 +272,18 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel):
yield self.format_tensor_name(gguf.MODEL_TENSOR.V_TOK_EMBD_SEP, suffix=""), data_torch[ids]
return
yield from super().modify_tensors(data_torch, name, bid)
+
+
+# MiniCPM-OCR 1.0: the 4.6 stack with model_type minicpm_ocr.
+
+@ModelBase.register("MiniCPMOcrForConditionalGeneration")
+@ModelBase.example("openbmb/MiniCPM-OCR-1.0")
+class MiniCPMOCR1_0TextModel(MiniCPMV4_6TextModel):
+ model_arch = gguf.MODEL_ARCH.QWEN35
+ no_mtp = True
+
+
+@ModelBase.register("MiniCPMOcrForConditionalGeneration")
+@ModelBase.example("openbmb/MiniCPM-OCR-1.0")
+class MiniCPMOCR1_0VisionModel(MiniCPMV4_6VisionModel):
+ projector_type = gguf.VisionProjectorType.MINICPM_OCR_1_0
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 0b95580bd..b8cfa3a85 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -6452,6 +6452,7 @@ class VisionProjectorType:
MINIMAXM3 = "minimax_m3"
MINICPMV4_6 = "minicpmv4_6"
MINICPMV4_7 = "minicpmv4_7"
+ MINICPM_OCR_1_0 = "minicpm_ocr_1_0"
GRANITE_SPEECH = "granite_speech" # audio
MIMOVL = "mimovl"
MIMO_AUDIO = "mimo_audio"
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 6ca465280..5583a7c57 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -537,6 +537,7 @@ enum projector_type {
PROJECTOR_TYPE_EXAONE4_5,
PROJECTOR_TYPE_MINICPMV4_6,
PROJECTOR_TYPE_MINICPMV4_7,
+ PROJECTOR_TYPE_MINICPM_OCR_1_0,
PROJECTOR_TYPE_GRANITE_SPEECH,
PROJECTOR_TYPE_MIMOVL,
PROJECTOR_TYPE_MINIMAX_M3,
@@ -611,6 +612,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"},
{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},
{ PROJECTOR_TYPE_MINICPMV4_7, "minicpmv4_7"},
+ { PROJECTOR_TYPE_MINICPM_OCR_1_0, "minicpm_ocr_1_0"},
{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},
{ PROJECTOR_TYPE_MIMOVL, "mimovl"},
{ PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"},
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 869e3fff8..5737400df 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1031,6 +1031,7 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
builder = std::make_unique<clip_graph_minicpmv4_6>(ctx, img);
} break;
@@ -1496,6 +1497,7 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
// MiniCPM-V 4.6/4.7 unified merger projector
// ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension
@@ -2527,6 +2529,7 @@ struct clip_model_loader {
|| model.proj_type == PROJECTOR_TYPE_MINICPMV
|| model.proj_type == PROJECTOR_TYPE_MINICPMV4_6
|| model.proj_type == PROJECTOR_TYPE_MINICPMV4_7
+ || model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0
) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd;
if (is_ffn_swapped) {
// swap up and down weights
@@ -2630,6 +2633,7 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
const bool merger_required = hparams.n_merge == 4;
auto get_merger_tensor = [&](const std::string & name, bool required = true) {
@@ -4348,6 +4352,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_STEP3VL:
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
return img->nx() / (params.patch_size * params.n_merge);
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
@@ -4379,6 +4384,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_STEP3VL:
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
return img->ny() / (params.patch_size * params.n_merge);
default:
break;
@@ -4445,6 +4451,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
n_patches /= params.n_merge * params.n_merge;
} break;
@@ -5169,6 +5176,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
const bool is_4x = hparams.n_merge == 2;
@@ -6437,6 +6445,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_model_proj->ne[0];
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
return ctx->model.mm_ffn_down_w->ne[1];
case PROJECTOR_TYPE_GLM_EDGE:
return ctx->model.mm_model_mlp_3_w->ne[1];
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index 3aea23ced..bc90a76c6 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -190,6 +190,17 @@ protected:
}
};
+// MiniCPM-OCR 1.0: find_best_resize aligns to patch_size * n_merge (28 in 4x).
+struct mtmd_image_preprocessor_minicpm_ocr_1_0 : mtmd_image_preprocessor_minicpmv {
+ using mtmd_image_preprocessor_minicpmv::mtmd_image_preprocessor_minicpmv;
+
+protected:
+ int get_slice_align() const override {
+ const int merge = hparams.n_merge > 0 ? hparams.n_merge : 2;
+ return hparams.patch_size * merge;
+ }
+};
+
// custom llava-uhd slicing logic for LFM2
// tiling params (min/max tiles, tile size, pixels tolerance) come from hparams
// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 9460f15fe..9a1663015 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -722,6 +722,7 @@ struct mtmd_context {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
slice_tmpl = MTMD_SLICE_TMPL_MINICPMV_2_6;
tok_ov_img_start = {lookup_token("<image>")};
@@ -732,7 +733,11 @@ struct mtmd_context {
tok_row_end_trail = false; // no trailing end-of-row token
ov_img_first = true;
use_image_id = true;
- image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);
+ if (clip_get_projector_type(ctx_v) == PROJECTOR_TYPE_MINICPM_OCR_1_0) {
+ image_preproc = std::make_unique<mtmd_image_preprocessor_minicpm_ocr_1_0>(ctx_v);
+ } else {
+ image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);
+ }
} break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL: