Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-04 02:37:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

mtmd: fix idefics3 preproc

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-09-02 23:56:21 +02:00
parent 9cffdcc801
commit 0349b3fbcb
1 changed files with 50 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/mtmd-image.cpp
+50
View File
@@ -980,6 +980,56 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
//
// CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737
const clip_image_size original_size = img.get_size();
// old gguf files have no preprocessor longest size, custom token limits also need the generic size below
if (hparams.image_longest_edge > 0 && hparams.image_min_pixels <= 0 && hparams.image_max_pixels <= 0) {
const int tile_size = hparams.image_size;
const int longest_edge = hparams.image_longest_edge;
const double aspect_ratio = (double) original_size.width / original_size.height;
clip_image_size resized_size;
if (original_size.width >= original_size.height) {
resized_size.width = longest_edge;
resized_size.height = (int) (longest_edge / aspect_ratio);
resized_size.height += resized_size.height % 2;
} else {
resized_size.height = longest_edge;
resized_size.width = (int) (longest_edge * aspect_ratio);
resized_size.width += resized_size.width % 2;
}
const int grid_x = (resized_size.width + tile_size - 1) / tile_size;
const int grid_y = (resized_size.height + tile_size - 1) / tile_size;
const clip_image_size refined_size = clip_image_size{grid_x * tile_size, grid_y * tile_size};
clip_image_u8 resized_img;
img_tool::resize(img, resized_img, resized_size, hparams.image_resize_algo, PAD_NONE);
clip_image_u8 refined_img;
img_tool::resize(resized_img, refined_img, refined_size, hparams.image_resize_algo, PAD_NONE);
clip_image_u8 overview;
img_tool::resize(refined_img, overview, {tile_size, tile_size}, hparams.image_resize_algo, PAD_NONE);
std::vector<clip_image_u8> slices;
for (int y = 0; y < grid_y; y++) {
for (int x = 0; x < grid_x; x++) {
clip_image_u8 slice;
img_tool::crop(refined_img, slice, x * tile_size, y * tile_size, tile_size, tile_size);
slices.push_back(std::move(slice));
}
}
LOG_DBG("%s: grid size: %d x %d (%d tiles) + overview\n", __func__, grid_x, grid_y, grid_x * grid_y);
mtmd_image_preproc_out output;
output.append_overview(hparams, overview, true);
output.append(hparams, slices, true);
output.grid_x = grid_x;
output.grid_y = grid_y;
return output;
}
const clip_image_size refined_size = img_tool::calc_size_preserved_ratio(
original_size,
{ hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge });