Commit 73c941b11 for llama.cpp
commit 73c941b11165cc0f7a36ba17380e79e8fe9797dc
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Tue Sep 22 17:53:05 2026 +0200
mtmd: add various sanity checks (#29276)
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 1783c7ddf..feceb7ff7 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1408,9 +1408,14 @@ struct clip_model_loader {
}
// Load the vision/audio feature layer indices if they are explicitly provided
- // NOTE: gguf conversions should standardize the values of the vision feature layer to
- // be non-negative, since we use -1 to mark values as unset here.
+ // NOTE: gguf conversions should standardize the values of the vision feature layer to be non-negative, since we use -1 to mark values as unset here.
get_arr_int(string_format(KEY_FEATURE_LAYERS, prefix), hparams.feature_layers, false);
+ for (const auto & v : hparams.feature_layers) {
+ if (v > (int) hparams.n_layer) {
+ throw std::runtime_error(string_format("%s: feature layer index %d is out of range (n_layer: %d)",
+ __func__, v, hparams.n_layer));
+ }
+ }
// model-specific params
switch (model.proj_type) {
@@ -1456,7 +1461,12 @@ struct clip_model_loader {
std::vector<int> wa_layer_indexes_vec;
get_arr_int(KEY_WIN_ATTN_LAYER_INDEXES, wa_layer_indexes_vec, false);
if (!wa_layer_indexes_vec.empty()) {
- hparams.insert_layer_id = wa_layer_indexes_vec[0];
+ const int insert_lid = wa_layer_indexes_vec[0];
+ if (insert_lid < 0 || insert_lid >= (int) hparams.n_layer) {
+ throw std::runtime_error(string_format("%s: layer index %d is out of range (n_layer: %d)",
+ __func__, insert_lid, hparams.n_layer));
+ }
+ hparams.insert_layer_id = insert_lid;
}
} break;
case PROJECTOR_TYPE_INTERNVL:
@@ -3226,6 +3236,7 @@ struct clip_model_loader {
model.pos_embed = get_tensor(string_format(TN_SAM_POS_EMBD, "weight"));
model.patch_embed_proj_w = get_tensor(string_format(TN_SAM_PATCH_EMBD, "weight"));
model.patch_embed_proj_b = get_tensor(string_format(TN_SAM_PATCH_EMBD, "bias"));
+ model.n_sam_layers = hparams.sam_n_layer;
model.sam_layers.resize(model.n_sam_layers);
for (int il = 0; il < model.n_sam_layers; ++il) {
auto & layer = model.sam_layers[il];
@@ -4652,8 +4663,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
// -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit
// -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316
std::vector<int32_t> positions(pos_h * pos_w);
- int bucket_coords_h[1024];
- int bucket_coords_w[1024];
+ // note: sized by the actual patch counts; a tall/wide image produces more
+ // than 1024 patches per side and a fixed [1024] array would be overrun
+ std::vector<int> bucket_coords_h(pos_h);
+ std::vector<int> bucket_coords_w(pos_w);
for (int i = 0; i < pos_h; i++){
bucket_coords_h[i] = std::floor(70.0*i/pos_h);
}
@@ -4696,8 +4709,8 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
// SigLIP position buckets (same as resampler path)
std::vector<int32_t> positions(pos_h * pos_w);
- int bucket_coords_h[1024];
- int bucket_coords_w[1024];
+ std::vector<int> bucket_coords_h(pos_h);
+ std::vector<int> bucket_coords_w(pos_w);
for (int i = 0; i < pos_h; i++){
bucket_coords_h[i] = std::floor(70.0*i/pos_h);
}
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index c11d35c87..9b2e98862 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -294,8 +294,8 @@ private:
support = filter_support * filterscale; // Widen filter when downsampling
ksize = static_cast<int>(std::ceil(support)) * 2 + 1; // Total pixels in kernel
- std::vector<double> pre_weights(outSize * ksize); // Temporary weights
- bounds.resize(outSize * 2);
+ std::vector<double> pre_weights((size_t) outSize * ksize); // Temporary weights
+ bounds.resize((size_t) outSize * 2);
// For each output pixel, compute its filter coefficients
@@ -322,20 +322,20 @@ private:
for (x = 0; x < xmax; x++) {
// Distance from input pixel center to output pixel center in input space
double w = resample_filter((x + xmin - center + 0.5) * ss);
- pre_weights[xx * ksize + x] = w;
+ pre_weights[(size_t) xx * ksize + x] = w;
ww += w; // Accumulate for normalization
}
// Normalize weights to sum to 1.0 (preserves brightness)
for (x = 0; x < xmax; x++) {
if (ww != 0.0) {
- pre_weights[xx * ksize + x] /= ww;
+ pre_weights[(size_t) xx * ksize + x] /= ww;
}
}
// Zero-pad remaining kernel positions
for (; x < ksize; x++) {
- pre_weights[xx * ksize + x] = 0;
+ pre_weights[(size_t) xx * ksize + x] = 0;
}
// Store input pixel range for this output pixel
@@ -345,11 +345,11 @@ private:
// Convert floating-point coefficients to fixed-point integers
// Formula: int32 = round(float * 2^PRECISION_BITS)
- weights.resize(outSize * ksize);
+ weights.resize((size_t) outSize * ksize);
const double fxp_scale = std::ldexp(1.0, PRECISION_BITS); // 1.0 * 2^PRECISION_BITS
- for (int i = 0; i < outSize * ksize; i++) {
+ for (size_t i = 0; i < (size_t) outSize * ksize; i++) {
// Pillow adds +/- 0.5 then truncates toward zero; std::round would round twice
const double rounded = pre_weights[i] * fxp_scale + (pre_weights[i] < 0 ? -0.5 : 0.5);
weights[i] = static_cast<int32_t>(rounded);
@@ -442,6 +442,12 @@ private:
const int src_width = img.get_size().width;
const int src_height = img.get_size().height;
+ // sanity check on the target size
+ if (target_width <= 0 || target_width > 65536 || target_height <= 0 || target_height > 65536) {
+ throw std::runtime_error("resize target " + std::to_string(target_width) + "x" +
+ std::to_string(target_height) + " is out of range (max 65536)");
+ }
+
bool need_horizontal = (target_width != src_width);
bool need_vertical = (target_height != src_height);