diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index ce08f9e931..c25bf4ec89 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -549,7 +549,7 @@ void mtmd_audio_preprocessor_whisper::initialize() { bool mtmd_audio_preprocessor_whisper::preprocess(const float * samples, size_t n_samples, - std::vector & output) { + std::vector & output) const { if (n_samples == 0) { // empty audio return false; @@ -637,7 +637,7 @@ void mtmd_audio_preprocessor_qwen3a::initialize() { bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * samples, size_t n_samples, - std::vector & output) { + std::vector & output) const { if (n_samples == 0) { return false; } @@ -739,7 +739,7 @@ void mtmd_audio_preprocessor_dots3note::initialize() { bool mtmd_audio_preprocessor_dots3note::preprocess(const float * samples, size_t n_samples, - std::vector & output) { + std::vector & output) const { if (n_samples == 0) { return false; } @@ -839,7 +839,7 @@ void mtmd_audio_preprocessor_mimo_audio::initialize() { bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float * samples, size_t n_samples, - std::vector & output) { + std::vector & output) const { if (n_samples == 0) { return false; } @@ -898,7 +898,7 @@ void mtmd_audio_preprocessor_qwen3tts_spk::initialize() { bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * samples, size_t n_samples, - std::vector & output) { + std::vector & output) const { if (n_samples == 0) { return false; } @@ -955,7 +955,7 @@ void mtmd_audio_preprocessor_conformer::initialize() { bool mtmd_audio_preprocessor_conformer::preprocess(const float * samples, size_t n_samples, - std::vector & output) { + std::vector & output) const { // empty audio if (n_samples == 0) { return false; @@ -1003,7 +1003,7 @@ void mtmd_audio_preprocessor_granite_speech::initialize() { bool mtmd_audio_preprocessor_granite_speech::preprocess(const float * samples, size_t n_samples, - std::vector & output) { + std::vector & output) const { if (n_samples == 0) { return false; } @@ -1117,7 +1117,7 @@ void mtmd_audio_preprocessor_gemma4a::initialize() { bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * samples, size_t n_samples, - std::vector & output) { + std::vector & output) const { if (n_samples == 0) { return false; } @@ -1266,7 +1266,7 @@ void mtmd_audio_preprocessor_parakeet::initialize() { bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples, size_t n_samples_in, - std::vector & output) { + std::vector & output) const { if (n_samples_in == 0) { return false; } @@ -1386,7 +1386,7 @@ void mtmd_audio_preprocessor_gemma4ua::initialize() { bool mtmd_audio_preprocessor_gemma4ua::preprocess(const float * samples, size_t n_samples, - std::vector & output) { + std::vector & output) const { if (n_samples == 0) { return false; } @@ -1527,7 +1527,7 @@ std::vector mtmd_audio_streaming_istft::flush() { bool mtmd_audio_preprocessor_pockettts::preprocess(const float * samples, size_t n_samples, - std::vector & output) { + std::vector & output) const { // the encoder needs whole frames, see pad_for_conv1d() in the reference const int64_t frame_size = (int64_t) hparams.mimi_downsample * 120; if (n_samples == 0 || frame_size <= 0) { diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index 0f47d45022..4a15fe6d4a 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -57,13 +57,13 @@ struct mtmd_audio_preprocessor { virtual ~mtmd_audio_preprocessor() = default; virtual void initialize() = 0; // NOT thread-safe - virtual bool preprocess(const float * samples, size_t n_samples, std::vector & output) = 0; + virtual bool preprocess(const float * samples, size_t n_samples, std::vector & output) const = 0; }; struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor { mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; private: mtmd_audio_cache cache; @@ -72,7 +72,7 @@ struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor { mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; private: mtmd_audio_cache cache; @@ -81,7 +81,7 @@ struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor { mtmd_audio_preprocessor_granite_speech(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; private: mtmd_audio_cache cache; @@ -90,7 +90,7 @@ struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor { mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; private: mtmd_audio_cache cache; @@ -99,13 +99,13 @@ struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_gemma4ua : mtmd_audio_preprocessor { mtmd_audio_preprocessor_gemma4ua(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; }; struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor { mtmd_audio_preprocessor_qwen3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; private: mtmd_audio_cache cache; @@ -114,7 +114,7 @@ struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor { mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; private: mtmd_audio_cache cache; @@ -123,7 +123,7 @@ struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; private: mtmd_audio_cache cache; @@ -132,7 +132,7 @@ struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor { mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; private: mtmd_audio_cache cache; @@ -142,13 +142,13 @@ struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor { mtmd_audio_preprocessor_pockettts(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override {} - bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; }; struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor { mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { } void initialize() override; - bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; private: mtmd_audio_cache cache; diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 890578978f..c11d35c87d 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -485,7 +485,7 @@ private: // mtmd_image_preprocessor_llava_uhd // -mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_image_u8 & img) const { const clip_image_size original_size = img.get_size(); auto const inst = get_slice_instructions(original_size); auto sliced = slice_image(img, inst); @@ -499,7 +499,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_ return output; } -mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_llava_uhd::get_slice_instructions(const clip_image_size & original_size) { +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_llava_uhd::get_slice_instructions(const clip_image_size & original_size) const { mtmd_image_preprocessor_llava_uhd::slice_instructions res; // align slices by patch_size * n_merge so an integer number of merger output tokens fits per slice const int n_merge = hparams.n_merge; @@ -604,7 +604,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_ll return res; } -mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uhd::slice_image(const clip_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst) { +mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uhd::slice_image(const clip_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst) const { slice_output output; // resize to overview size @@ -636,7 +636,7 @@ mtmd_image_preprocessor_llava_uhd::slice_output mtmd_image_preprocessor_llava_uh return output; } -clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale) { +clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale) const { int width = original_size.width; int height = original_size.height; if ((width * height > scale_resolution * scale_resolution) || allow_upscale) { @@ -650,7 +650,7 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_im return res; } -clip_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) { +clip_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) const { float scale_width = static_cast(target_max.width) / orig.width; float scale_height = static_cast(target_max.height) / orig.height; float scale = std::min(scale_width, scale_height); @@ -660,7 +660,7 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio( }; } -clip_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const clip_image_size & original_size, const std::vector & possible_resolutions) { +clip_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const clip_image_size & original_size, const std::vector & possible_resolutions) const { clip_image_size best_fit; int min_wasted_area = std::numeric_limits::max(); int max_effective_resolution = 0; @@ -684,11 +684,11 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const return best_fit; } -int mtmd_image_preprocessor_llava_uhd::ensure_divide(int length, int patch_size) { +int mtmd_image_preprocessor_llava_uhd::ensure_divide(int length, int patch_size) const { return std::max(static_cast(std::round(static_cast(length) / patch_size) * patch_size), patch_size); } -clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale) { +clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale) const { int width = original_size.width; int height = original_size.height; int grid_x = grid.width; @@ -711,7 +711,7 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_im return refine_size; } -clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) { +clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const { std::vector candidate_split_grids_nums; for (int i : {multiple - 1, multiple, multiple + 1}) { if (i == 1 || i > max_slice_nums) { @@ -747,7 +747,7 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_s // mtmd_image_preprocessor_fixed_size // -mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip_image_u8 & img) const { clip_image_u8 resized_image; int sz = hparams.image_size; img_tool::resize(img, resized_image, {sz, sz}, @@ -763,7 +763,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip // mtmd_image_preprocessor_dyn_size // -mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const { GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0); clip_image_u8 resized_image; const clip_image_size original_size = img.get_size(); @@ -790,7 +790,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i // mtmd_image_preprocessor_longest_edge // -mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img) const { GGML_ASSERT(hparams.image_longest_edge > 0); clip_image_u8 resized_image; const clip_image_size original_size = img.get_size(); @@ -817,7 +817,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const cl // mtmd_image_preprocessor_minicpmv // -mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const clip_image_size & original_size) { +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const clip_image_size & original_size) const { if (hparams.n_merge == 2) { const int slice_size = hparams.image_size; const float ratio = (float)original_size.width * original_size.height / (slice_size * slice_size); @@ -837,7 +837,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_mi // mtmd_image_preprocessor_lfm2 // -mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image_u8 & img) const { auto const inst = get_slice_instructions(img.get_size()); if (!inst.slices.empty()) { return mtmd_image_preprocessor_llava_uhd::preprocess(img); @@ -868,7 +868,7 @@ bool mtmd_image_preprocessor_lfm2::should_tile( static_cast(hparams.image_max_pixels) * max_pixels_tolerance; } -mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) { +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) const { mtmd_image_preprocessor_llava_uhd::slice_instructions inst; const int align_size = hparams.patch_size * hparams.n_merge; inst.overview_size = img_tool::calc_size_preserved_ratio( @@ -914,7 +914,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio( float aspect_ratio, const std::vector & target_ratios, - int width, int height) { + int width, int height) const { float best_ratio_diff = std::numeric_limits::max(); clip_image_size best_ratio = {1, 1}; const float area = static_cast(width * height); @@ -935,7 +935,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio( return best_ratio; } -std::vector mtmd_image_preprocessor_lfm2::get_target_ratios() { +std::vector mtmd_image_preprocessor_lfm2::get_target_ratios() const { std::vector ratios; for (int n = min_tiles; n <= max_tiles; n++) { for (int w = 1; w <= n; w++) { @@ -961,7 +961,7 @@ std::vector mtmd_image_preprocessor_lfm2::get_target_ratios() { return ratios; } -clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(int height, int width) { +clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(int height, int width) const { const float aspect_ratio = static_cast(width) / height; const auto ratios = get_target_ratios(); return find_closest_aspect_ratio(aspect_ratio, ratios, width, height); @@ -971,7 +971,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(int height, int wi // mtmd_image_preprocessor_idefics3 // -mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_image_u8 & img) const { // The refined size has two steps: // 1. Resize w/ aspect-ratio preserving such that the longer side is // the preprocessor longest size @@ -1071,7 +1071,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i // mtmd_image_preprocessor_internvl // -mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img) const { GGML_ASSERT(!hparams.image_res_candidates.empty()); const clip_image_size original_size = img.get_size(); auto const inst = get_slice_instructions(original_size); @@ -1206,7 +1206,7 @@ void mtmd_image_preprocessor_deepseek4v::safe_resize(int height, int width, int } // ref: load_image() -mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip_image_u8 & img) const { mtmd_image_preproc_out out; const int p = hparams.patch_size; @@ -1244,7 +1244,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip return out; } -mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) const { mtmd_image_preproc_out output; int grid_w = 0; int grid_h = 0; @@ -1320,7 +1320,7 @@ void mtmd_image_preprocessor_step3vl::img_u8_resize_bilinear_to_f32( int target_width, int target_height, const float mean[3], - const float std[3]) { + const float std[3]) const { const auto src_size = src.get_size(); if (src_size.width == target_width && src_size.height == target_height) { dst.from_u8(src); @@ -1519,7 +1519,7 @@ mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step return instructions; } -mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_image_u8 & img) const { clip_image_u8 prepared = prepare_image(img, hparams); const auto instructions = build_slice_instructions(hparams, prepared.get_size()); @@ -1573,7 +1573,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_im // mtmd_image_preprocessor_youtuvl // -mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_image_u8 & img) const { const int patch_size = hparams.patch_size; // typically 16 const int merge_size = hparams.n_merge; // typically 2 const int align_size = patch_size * merge_size; // 32 @@ -1622,7 +1622,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_im return output; } -mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img) const { GGML_ASSERT(!hparams.image_res_candidates.empty()); const clip_image_size orig_size = img.get_size(); @@ -1717,7 +1717,7 @@ static clip_image_size muse_glimmer_grid_size(int img_w, int img_h, int patch_hw return clip_image_size{ best_npw * patch_hw, best_nph * patch_hw }; } -mtmd_image_preproc_out mtmd_image_preprocessor_muse_glimmer::preprocess(const clip_image_u8 & img) { +mtmd_image_preproc_out mtmd_image_preprocessor_muse_glimmer::preprocess(const clip_image_u8 & img) const { const int patch_hw = hparams.patch_size * hparams.n_merge; const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; GGML_ASSERT(patch_area > 0 && hparams.image_max_pixels > 0); diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 8758c66472..e2cf698723 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -33,7 +33,7 @@ struct mtmd_image_preprocessor { mtmd_image_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {} virtual ~mtmd_image_preprocessor() = default; - virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) = 0; + virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const = 0; }; /** @@ -59,7 +59,7 @@ struct mtmd_image_preprocessor { */ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { mtmd_image_preprocessor_llava_uhd(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; struct slice_coordinates { int x; @@ -74,16 +74,16 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { std::vector slices; }; - virtual slice_instructions get_slice_instructions(const clip_image_size & original_size); + virtual slice_instructions get_slice_instructions(const clip_image_size & original_size) const; struct slice_output { clip_image_u8 overview; std::vector slices; }; - slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst); + slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst) const; protected: - clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false); + clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false) const; /** * Selects the best resolution from a list of possible resolutions based on the original size. @@ -100,19 +100,19 @@ protected: * @param possible_resolutions A list of possible resolutions * @return The best fit resolution */ - clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector & possible_resolutions); + clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector & possible_resolutions) const; private: - clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max); - int ensure_divide(int length, int patch_size); - clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false); - clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio); + clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) const; + int ensure_divide(int length, int patch_size) const; + clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false) const; + clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const; }; // downscale or upscale the input image to fixed size struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor { mtmd_image_preprocessor_fixed_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; }; // resize image to multiple of patch_size*n_merge, while preserving aspect ratio @@ -120,19 +120,19 @@ struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor { // this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor { mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; }; // similar to mtmd_image_preprocessor_dyn_size, but resize the image to have longest edge equal to hparams.image_longest_edge, while preserving aspect ratio struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor { mtmd_image_preprocessor_longest_edge(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; }; // ref: inference/image_processor.py in the HF repo (DeepSeek-V4-Flash-Vision) struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor { mtmd_image_preprocessor_deepseek4v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; private: struct grid_info { @@ -148,7 +148,7 @@ private: // custom llava-uhd slicing logic for MiniCPM-V struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd { using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd; - slice_instructions get_slice_instructions(const clip_image_size & original_size) override; + slice_instructions get_slice_instructions(const clip_image_size & original_size) const override; }; // custom llava-uhd slicing logic for LFM2 @@ -161,8 +161,8 @@ struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd { static constexpr int tile_size = 512; using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd; - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; - slice_instructions get_slice_instructions(const clip_image_size & original_size) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; + slice_instructions get_slice_instructions(const clip_image_size & original_size) const override; static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size); @@ -170,19 +170,19 @@ private: clip_image_size find_closest_aspect_ratio( float aspect_ratio, const std::vector & target_ratios, - int width, int height); - std::vector get_target_ratios(); - clip_image_size get_grid_layout(int height, int width); + int width, int height) const; + std::vector get_target_ratios() const; + clip_image_size get_grid_layout(int height, int width) const; }; struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd { mtmd_image_preprocessor_idefics3(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; }; struct mtmd_image_preprocessor_internvl : mtmd_image_preprocessor_llava_uhd { mtmd_image_preprocessor_internvl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; }; // DeepSeek-OCR (v1/v2) global view + optional local tile grid @@ -194,7 +194,7 @@ struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor { tile_size(hparams.preproc_tile_size), min_tiles(hparams.preproc_min_tiles), max_tiles(hparams.preproc_max_tiles) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; private: bool fuse_row; // v1 fuses a tile-row into one image; v2 keeps tiles separate @@ -214,7 +214,7 @@ private: // ref: https://huggingface.co/stepfun-ai/Step3-VL-10B/blob/main/processing_step3.py struct mtmd_image_preprocessor_step3vl : mtmd_image_preprocessor_llava_uhd { mtmd_image_preprocessor_step3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; static slice_instructions build_slice_instructions(const clip_hparams & params, const clip_image_size & prepared_size); private: @@ -230,7 +230,7 @@ private: int target_width, int target_height, const float mean[3], - const float std[3]); + const float std[3]) const; static int get_image_longest_edge(const clip_hparams & params); static int determine_window_size(const clip_hparams & params, int longer, int shorter); static int calc_crop_extent(int length, int window_size); @@ -241,17 +241,17 @@ private: struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor { mtmd_image_preprocessor_youtuvl(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; }; // llava-next "anyres": stacks the overview and all tiles into one image, assembled by clip in a single graph struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd { mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; }; // pick the patch grid closest to the input aspect ratio under the per-image token cap, stretch-resize. struct mtmd_image_preprocessor_muse_glimmer : mtmd_image_preprocessor { mtmd_image_preprocessor_muse_glimmer(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; };