Commit 36140dbee for llama.cpp
commit 36140dbee26704d517889db16caf5318039beaeb
Author: Tarek Dakhran <tarek@liquid.ai>
Date: Tue Oct 20 16:05:30 2026 +0200
mtmd : read LFM2 tiling params from GGUF metadata (#30840)
* mtmd : read LFM2 tiling params from GGUF metadata
Write min_tiles, max_tiles, tile_size and max_pixels_tolerance from
processor_config.json when converting LFM2-VL mmproj, read them in clip.cpp
with the previous constants as defaults. do_image_splitting=false maps to
min_tiles=max_tiles=1 (no tiling).
Rebased version of #25524, original implementation by Michael DM (@michael-dm).
Co-authored-by: Michael DM <usable_siren180@simplelogin.com>
* fix test
---------
Co-authored-by: Michael DM <usable_siren180@simplelogin.com>
diff --git a/conversion/lfm2.py b/conversion/lfm2.py
index 113c91116..b1f0d1d32 100644
--- a/conversion/lfm2.py
+++ b/conversion/lfm2.py
@@ -374,6 +374,16 @@ class LFM2VLModel(MmprojModel):
if resize_algo is None:
raise ValueError(f"unsupported resample: {resample}")
self.gguf_writer.add_vision_image_resize_algo(resize_algo)
+ # tiling params from processor_config.json (do_image_splitting=False -> single tile, i.e. no tiling)
+ preproc = self.preprocessor_config
+ if preproc.get("do_image_splitting", True):
+ self.gguf_writer.add_vision_preproc_min_tiles(preproc.get("min_tiles", 2))
+ self.gguf_writer.add_vision_preproc_max_tiles(preproc.get("max_tiles", 10))
+ else:
+ self.gguf_writer.add_vision_preproc_min_tiles(1)
+ self.gguf_writer.add_vision_preproc_max_tiles(1)
+ self.gguf_writer.add_vision_preproc_tile_size(preproc.get("tile_size", 512))
+ self.gguf_writer.add_vision_preproc_max_pixels_tolerance(preproc.get("max_pixels_tolerance", 2.0))
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 8c19e22da..b2d8d59e0 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -419,6 +419,8 @@ class Keys:
PREPROC_MIN_TILES = "clip.vision.preproc_min_tiles"
PREPROC_MAX_TILES = "clip.vision.preproc_max_tiles"
PREPROC_IMAGE_SIZE = "clip.vision.preproc_image_size"
+ PREPROC_TILE_SIZE = "clip.vision.preproc_tile_size"
+ PREPROC_MAX_PIXELS_TOLERANCE = "clip.vision.preproc_max_pixels_tolerance"
PATCH_SIZE = "clip.vision.patch_size"
EMBEDDING_LENGTH = "clip.vision.embedding_length"
FEED_FORWARD_LENGTH = "clip.vision.feed_forward_length"
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 7becd74af..2943e0186 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1470,6 +1470,12 @@ class GGUFWriter:
def add_vision_preproc_image_size(self, value: int) -> None:
self.add_uint32(Keys.ClipVision.PREPROC_IMAGE_SIZE, value)
+ def add_vision_preproc_tile_size(self, value: int) -> None:
+ self.add_uint32(Keys.ClipVision.PREPROC_TILE_SIZE, value)
+
+ def add_vision_preproc_max_pixels_tolerance(self, value: float) -> None:
+ self.add_float32(Keys.ClipVision.PREPROC_MAX_PIXELS_TOLERANCE, value)
+
def add_vision_projector_query_side(self, value: int) -> None:
self.add_uint32(Keys.ClipVision.Projector.QUERY_SIDE, value)
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
index 2d0f24874..1b7796d95 100644
--- a/tests/test-mtmd-impl.cpp
+++ b/tests/test-mtmd-impl.cpp
@@ -96,6 +96,10 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
hparams.patch_size = 16;
hparams.n_merge = 2;
hparams.set_limit_image_tokens(64, 256);
+ hparams.preproc_min_tiles = 2;
+ hparams.preproc_max_tiles = 10;
+ hparams.preproc_tile_size = 512;
+ hparams.preproc_max_pixels_tolerance = 2.0f;
// { image size, expected tiling }
const std::vector<std::pair<clip_image_size, bool>> cases = {
@@ -116,6 +120,12 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
std::string(expected ? "tiled" : "single"),
std::string(actual ? "tiled" : "single"));
}
+
+ // do_image_splitting=false
+ hparams.preproc_min_tiles = 1;
+ hparams.preproc_max_tiles = 1;
+ t.assert_equal("tiling for 1024x977 with max_tiles=1", std::string("single"),
+ std::string(mtmd_image_preprocessor_lfm2::should_tile(hparams, { 1024, 977 }) ? "tiled" : "single"));
}
//
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 10eea6f74..8c4ffc9bb 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -56,6 +56,8 @@
#define KEY_PREPROC_MIN_TILES "clip.vision.preproc_min_tiles"
#define KEY_PREPROC_MAX_TILES "clip.vision.preproc_max_tiles"
#define KEY_PREPROC_IMAGE_SIZE "clip.vision.preproc_image_size"
+#define KEY_PREPROC_TILE_SIZE "clip.vision.preproc_tile_size"
+#define KEY_PREPROC_MAX_PIXELS_TOLERANCE "clip.vision.preproc_max_pixels_tolerance"
#define KEY_PATCH_SIZE "clip.vision.patch_size"
#define KEY_IMAGE_MEAN "clip.vision.image_mean"
#define KEY_IMAGE_STD "clip.vision.image_std"
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index fefbd4f46..64e006e8d 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -73,7 +73,8 @@ struct clip_hparams {
int32_t preproc_min_tiles = 0;
int32_t preproc_max_tiles = 0;
int32_t max_slice_nums = 9; // llava-uhd slice cap; per-model, carried in the GGUF
- int32_t preproc_tile_size = 0; // local tile size (deepseek-ocr)
+ int32_t preproc_tile_size = 0; // local tile size (deepseek-ocr, lfm2)
+ float preproc_max_pixels_tolerance = 1.0f;
resize_algo image_resize_algo_rf = RESIZE_ALGO_BICUBIC;
resize_algo image_resize_algo_ov = RESIZE_ALGO_BICUBIC;
pad_style image_pad_rf = PAD_CEIL; // padding style for the refined image (e.g. llava-1.6)
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index faeb37702..3a4dd682a 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1598,7 +1598,17 @@ struct clip_model_loader {
hparams.image_pad_rf = PAD_NONE;
}
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
+ // defaults for GGUFs converted before tiling params were written
// ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json
+ hparams.preproc_min_tiles = 2;
+ hparams.preproc_max_tiles = 10;
+ hparams.preproc_tile_size = 512;
+ hparams.preproc_max_pixels_tolerance = 2.0f;
+ get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false);
+ get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false);
+ get_u32(KEY_PREPROC_TILE_SIZE, hparams.preproc_tile_size, false);
+ get_f32(KEY_PREPROC_MAX_PIXELS_TOLERANCE, hparams.preproc_max_pixels_tolerance, false);
+ GGML_ASSERT(hparams.preproc_min_tiles >= 1 && hparams.preproc_min_tiles <= hparams.preproc_max_tiles);
hparams.set_limit_image_tokens(64, 256);
} break;
case PROJECTOR_TYPE_PHI4:
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index e56f33e77..86feea6bd 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -947,8 +947,9 @@ bool mtmd_image_preprocessor_lfm2::should_tile(
const int h_bar = std::max(hparams.patch_size, round_by_factor(original_size.height));
const int w_bar = std::max(hparams.patch_size, round_by_factor(original_size.width));
- return static_cast<double>(h_bar) * static_cast<double>(w_bar) >
- static_cast<double>(hparams.image_max_pixels) * max_pixels_tolerance;
+ return hparams.preproc_max_tiles > 1 &&
+ static_cast<double>(h_bar) * static_cast<double>(w_bar) >
+ static_cast<double>(hparams.image_max_pixels) * hparams.preproc_max_pixels_tolerance;
}
mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) const {
@@ -959,6 +960,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf
{ align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 });
const bool needs_tiling = should_tile(hparams, original_size);
+ const int tile_size = hparams.preproc_tile_size;
if (!needs_tiling) {
inst.refined_size = clip_image_size{0, 0};
@@ -1009,6 +1011,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(
best_ratio_diff = ratio_diff;
best_ratio = ratio;
} else if (ratio_diff == best_ratio_diff) {
+ const int tile_size = hparams.preproc_tile_size;
const float target_area = static_cast<float>(tile_size * tile_size * ratio.width * ratio.height);
if (area > 0.5f * target_area) {
best_ratio = ratio;
@@ -1019,6 +1022,8 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(
}
std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios() const {
+ const int min_tiles = hparams.preproc_min_tiles;
+ const int max_tiles = hparams.preproc_max_tiles;
std::vector<clip_image_size> ratios;
for (int n = min_tiles; n <= max_tiles; n++) {
for (int w = 1; w <= n; w++) {
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index 26193e4e4..a9d16f1c9 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -191,14 +191,9 @@ protected:
};
// custom llava-uhd slicing logic for LFM2
+// tiling params (min/max tiles, tile size, pixels tolerance) come from hparams
// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py
struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {
- // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json
- static constexpr int min_tiles = 2;
- static constexpr int max_tiles = 10;
- static constexpr float max_pixels_tolerance = 2.0f;
- static constexpr int tile_size = 512;
-
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;