From 0349b3fbcb69e31f74f222ba6e797b7a67b9cbce Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 2 Sep 2026 23:56:21 +0200 Subject: [PATCH] mtmd: fix idefics3 preproc --- tools/mtmd/mtmd-image.cpp | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 65c24f4d47..890578978f 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -980,6 +980,56 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i // // CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737 const clip_image_size original_size = img.get_size(); + + // old gguf files have no preprocessor longest size, custom token limits also need the generic size below + if (hparams.image_longest_edge > 0 && hparams.image_min_pixels <= 0 && hparams.image_max_pixels <= 0) { + const int tile_size = hparams.image_size; + const int longest_edge = hparams.image_longest_edge; + const double aspect_ratio = (double) original_size.width / original_size.height; + + clip_image_size resized_size; + if (original_size.width >= original_size.height) { + resized_size.width = longest_edge; + resized_size.height = (int) (longest_edge / aspect_ratio); + resized_size.height += resized_size.height % 2; + } else { + resized_size.height = longest_edge; + resized_size.width = (int) (longest_edge * aspect_ratio); + resized_size.width += resized_size.width % 2; + } + + const int grid_x = (resized_size.width + tile_size - 1) / tile_size; + const int grid_y = (resized_size.height + tile_size - 1) / tile_size; + const clip_image_size refined_size = clip_image_size{grid_x * tile_size, grid_y * tile_size}; + + clip_image_u8 resized_img; + img_tool::resize(img, resized_img, resized_size, hparams.image_resize_algo, PAD_NONE); + + clip_image_u8 refined_img; + img_tool::resize(resized_img, refined_img, refined_size, hparams.image_resize_algo, PAD_NONE); + + clip_image_u8 overview; + img_tool::resize(refined_img, overview, {tile_size, tile_size}, hparams.image_resize_algo, PAD_NONE); + + std::vector slices; + for (int y = 0; y < grid_y; y++) { + for (int x = 0; x < grid_x; x++) { + clip_image_u8 slice; + img_tool::crop(refined_img, slice, x * tile_size, y * tile_size, tile_size, tile_size); + slices.push_back(std::move(slice)); + } + } + + LOG_DBG("%s: grid size: %d x %d (%d tiles) + overview\n", __func__, grid_x, grid_y, grid_x * grid_y); + + mtmd_image_preproc_out output; + output.append_overview(hparams, overview, true); + output.append(hparams, slices, true); + output.grid_x = grid_x; + output.grid_y = grid_y; + return output; + } + const clip_image_size refined_size = img_tool::calc_size_preserved_ratio( original_size, { hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge });