Commit 4a45de4f2 for llama.cpp
commit 4a45de4f2c68f3d144f6c2431ad7c1da7e53c99e
Author: Yuri Khrustalev <ykhrustalev@users.noreply.github.com>
Date: Tue Oct 20 12:54:46 2026 -0400
mtmd : lfm2 tiles and layout match transformers (#30261)
* mtmd : lfm2 tiles stretch the image to the grid
Lfm2VlImageProcessor.crop_image_to_patches resizes the image straight to
(512 * rows, 512 * cols) before it cuts the tiles. llama.cpp kept the aspect
ratio and added black bars for LFM2 (only d1-omni stretched), so the tiles of
d1-3B and every LFM2-VL / LFM2.5-VL model differed from transformers unless
the image had the aspect ratio of the grid. With PAD_NONE the tiles are the
same as in transformers.
* mtmd : lfm2 layout with Python rounding in double precision
Lfm2VlImageProcessor.smart_resize rounds with Python's round (half to even)
and computes beta and the tile grid aspect ratio in double. llama.cpp used
std::round and float, so 31926 of the 16.7M sizes in [1,4096]^2 got another
thumbnail size or grid than transformers (and d1-omni vision.py), e.g.
400x600 -> 416x608 instead of 384x608 and 720x1600 -> 2x5 tiles instead of
2x4. The lfm2 smart resize and grid choice now follow the Python code and
match it on every size of that range.
* fix
---------
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
index 1b7796d95..6ad60213e 100644
--- a/tests/test-mtmd-impl.cpp
+++ b/tests/test-mtmd-impl.cpp
@@ -91,6 +91,10 @@ MAKE_TEST(test_image_preprocessor_dyn_size) {
}
}
+static std::string image_size_to_str(const clip_image_size & size) {
+ return std::to_string(size.width) + "x" + std::to_string(size.height);
+}
+
MAKE_TEST(test_image_preprocessor_lfm2) {
clip_hparams hparams;
hparams.patch_size = 16;
@@ -121,6 +125,32 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
std::string(actual ? "tiled" : "single"));
}
+ // { image size, expected size from HF smart_resize }
+ const std::vector<std::pair<clip_image_size, clip_image_size>> resize_cases = {
+ { { 400, 600 }, { 384, 608 } }, // 400 / 32 = 12.5 rounds half to even
+ { { 480, 336 }, { 480, 320 } },
+ { { 64, 169 }, { 160, 416 } }, // 169 * beta / 32 is exactly 13 in double
+ { { 1024, 768 }, { 576, 416 } },
+ };
+
+ for (const auto & [size, expected] : resize_cases) {
+ t.assert_equal("smart_resize for " + image_size_to_str(size), image_size_to_str(expected),
+ image_size_to_str(mtmd_image_preprocessor_lfm2::smart_resize(hparams, size)));
+ }
+
+ // { image size, expected grid from HF }
+ const std::vector<std::pair<clip_image_size, clip_image_size>> grid_cases = {
+ { { 720, 1600 }, { 2, 4 } }, // 0.45 is an exact tie between 2/4 and 2/5 in double, not in float
+ { { 420, 1440 }, { 1, 3 } },
+ { { 255, 2160 }, { 1, 8 } },
+ { { 1024, 768 }, { 3, 2 } },
+ };
+
+ for (const auto & [size, expected] : grid_cases) {
+ t.assert_equal("grid for " + image_size_to_str(size), image_size_to_str(expected),
+ image_size_to_str(mtmd_image_preprocessor_lfm2::get_grid_layout(hparams, size.height, size.width)));
+ }
+
// do_image_splitting=false
hparams.preproc_min_tiles = 1;
hparams.preproc_max_tiles = 1;
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 41aba280b..26941c9a8 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1593,10 +1593,8 @@ struct clip_model_loader {
}
hparams.image_resize_algo_rf = hparams.image_resize_algo;
hparams.image_resize_algo_ov = hparams.image_resize_algo;
- // the tiles stretch the image to the grid (d1-omni vision.py)
- if (model.proj_type == PROJECTOR_TYPE_D1OMNI_V) {
- hparams.image_pad_rf = PAD_NONE;
- }
+ // the tiles stretch the image to the grid (Lfm2VlImageProcessor, d1-omni vision.py)
+ hparams.image_pad_rf = PAD_NONE;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
// defaults for GGUFs converted before tiling params were written
// ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index 86feea6bd..e0cd0bf1d 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -952,12 +952,33 @@ bool mtmd_image_preprocessor_lfm2::should_tile(
static_cast<double>(hparams.image_max_pixels) * hparams.preproc_max_pixels_tolerance;
}
+// same as Lfm2VlImageProcessor.smart_resize: Python's round half to even, double precision
+clip_image_size mtmd_image_preprocessor_lfm2::smart_resize(
+ const clip_hparams & hparams,
+ const clip_image_size & original_size) {
+ const int align_size = hparams.patch_size * hparams.n_merge;
+ const double height = original_size.height;
+ const double width = original_size.width;
+
+ int h_bar = std::max(align_size, static_cast<int>(std::nearbyint(height / align_size)) * align_size);
+ int w_bar = std::max(align_size, static_cast<int>(std::nearbyint(width / align_size)) * align_size);
+
+ if (static_cast<double>(h_bar) * w_bar > hparams.image_max_pixels) {
+ const double beta = std::sqrt(height * width / hparams.image_max_pixels);
+ h_bar = std::max(align_size, static_cast<int>(std::floor(height / beta / align_size)) * align_size);
+ w_bar = std::max(align_size, static_cast<int>(std::floor(width / beta / align_size)) * align_size);
+ } else if (static_cast<double>(h_bar) * w_bar < hparams.image_min_pixels) {
+ const double beta = std::sqrt(hparams.image_min_pixels / (height * width));
+ h_bar = static_cast<int>(std::ceil(height * beta / align_size)) * align_size;
+ w_bar = static_cast<int>(std::ceil(width * beta / align_size)) * align_size;
+ }
+
+ return {w_bar, h_bar};
+}
+
mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) const {
mtmd_image_preprocessor_llava_uhd::slice_instructions inst;
- const int align_size = hparams.patch_size * hparams.n_merge;
- inst.overview_size = img_tool::calc_size_preserved_ratio(
- original_size,
- { align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 });
+ inst.overview_size = smart_resize(hparams, original_size);
const bool needs_tiling = should_tile(hparams, original_size);
const int tile_size = hparams.preproc_tile_size;
@@ -968,7 +989,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf
return inst;
}
- const clip_image_size grid = get_grid_layout(original_size.height, original_size.width);
+ const clip_image_size grid = get_grid_layout(hparams, original_size.height, original_size.width);
inst.grid_size = grid;
inst.refined_size = clip_image_size{tile_size * grid.width, tile_size * grid.height};
@@ -996,24 +1017,26 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf
return inst;
}
+// double precision as in Python, float breaks exact ties such as 720/1600 between 2/4 and 2/5
clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(
- float aspect_ratio,
+ const clip_hparams & hparams,
+ double aspect_ratio,
const std::vector<clip_image_size> & target_ratios,
- int width, int height) const {
- float best_ratio_diff = std::numeric_limits<float>::max();
+ int width, int height) {
+ double best_ratio_diff = std::numeric_limits<double>::max();
clip_image_size best_ratio = {1, 1};
- const float area = static_cast<float>(width * height);
+ const double area = static_cast<double>(width) * height;
for (const auto & ratio : target_ratios) {
- const float target_aspect_ratio = static_cast<float>(ratio.width) / ratio.height;
- const float ratio_diff = std::abs(aspect_ratio - target_aspect_ratio);
+ const double target_aspect_ratio = static_cast<double>(ratio.width) / ratio.height;
+ const double ratio_diff = std::abs(aspect_ratio - target_aspect_ratio);
if (ratio_diff < best_ratio_diff) {
best_ratio_diff = ratio_diff;
best_ratio = ratio;
} else if (ratio_diff == best_ratio_diff) {
- const int tile_size = hparams.preproc_tile_size;
- const float target_area = static_cast<float>(tile_size * tile_size * ratio.width * ratio.height);
- if (area > 0.5f * target_area) {
+ const int tile_size = hparams.preproc_tile_size;
+ const double target_area = static_cast<double>(tile_size * tile_size * ratio.width * ratio.height);
+ if (area > 0.5 * target_area) {
best_ratio = ratio;
}
}
@@ -1021,7 +1044,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(
return best_ratio;
}
-std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios() const {
+std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios(const clip_hparams & hparams) {
const int min_tiles = hparams.preproc_min_tiles;
const int max_tiles = hparams.preproc_max_tiles;
std::vector<clip_image_size> ratios;
@@ -1049,10 +1072,10 @@ std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios() c
return ratios;
}
-clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(int height, int width) const {
- const float aspect_ratio = static_cast<float>(width) / height;
- const auto ratios = get_target_ratios();
- return find_closest_aspect_ratio(aspect_ratio, ratios, width, height);
+clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const clip_hparams & hparams, int height, int width) {
+ const double aspect_ratio = static_cast<double>(width) / height;
+ const auto ratios = get_target_ratios(hparams);
+ return find_closest_aspect_ratio(hparams, aspect_ratio, ratios, width, height);
}
//
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index a9d16f1c9..3aea23ced 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -199,14 +199,16 @@ struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {
slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;
static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size);
+ static clip_image_size smart_resize(const clip_hparams & hparams, const clip_image_size & original_size);
+ static clip_image_size get_grid_layout(const clip_hparams & hparams, int height, int width);
private:
- clip_image_size find_closest_aspect_ratio(
- float aspect_ratio,
+ static clip_image_size find_closest_aspect_ratio(
+ const clip_hparams & hparams,
+ double aspect_ratio,
const std::vector<clip_image_size> & target_ratios,
- int width, int height) const;
- std::vector<clip_image_size> get_target_ratios() const;
- clip_image_size get_grid_layout(int height, int width) const;
+ int width, int height);
+ static std::vector<clip_image_size> get_target_ratios(const clip_hparams & hparams);
};
struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd {