From f830688e91214cccfa25ed0a2b9a708ee5a855c3 Mon Sep 17 00:00:00 2001 From: Toby <25832191+aetherbird@users.noreply.github.com> Date: Thu, 24 Sep 2026 02:57:31 -0400 Subject: [PATCH] model : add Ling 3.0 VL support (#29151) * model : fold Ling 3.0 VL into the BailingMoeV3 architecture Assisted-by: Scout * model : keep shared NORM rope list intact when gating bailingmoe3 on mrope sections --------- Co-authored-by: aetherbird --- conversion/__init__.py | 2 + conversion/bailingmoe3.py | 114 +++++++++++++++++++++++++++++++++- gguf-py/gguf/constants.py | 3 + src/llama-model.cpp | 5 +- src/models/bailingmoe3.cpp | 39 +++++++++--- tests/test-llama-archs.cpp | 8 ++- tools/mtmd/CMakeLists.txt | 1 + tools/mtmd/clip-impl.h | 2 + tools/mtmd/clip.cpp | 21 +++++++ tools/mtmd/models/ling3vl.cpp | 86 +++++++++++++++++++++++++ tools/mtmd/models/models.h | 5 ++ tools/mtmd/mtmd.cpp | 1 + 12 files changed, 275 insertions(+), 12 deletions(-) create mode 100644 tools/mtmd/models/ling3vl.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index f966373f1d..85db1d6431 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -28,6 +28,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "BailingMoeForCausalLM": "bailingmoe", "BailingMoeV2ForCausalLM": "bailingmoe", "BailingMoeV3ForCausalLM": "bailingmoe3", + "BailingMoeV3VLForConditionalGeneration": "bailingmoe3", "BambaForCausalLM": "granite", "BertForMaskedLM": "bert", "BertForSequenceClassification": "bert", @@ -302,6 +303,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Gemma4ForConditionalGeneration": "gemma", "Gemma4UnifiedForConditionalGeneration": "gemma", "Glm4vForConditionalGeneration": "qwen3vl", + "BailingMoeV3VLForConditionalGeneration": "bailingmoe3", "Glm4vMoeForConditionalGeneration": "qwen3vl", "Glm5vForConditionalGeneration": "kimivl", "GlmOcrForConditionalGeneration": "qwen3vl", diff --git a/conversion/bailingmoe3.py b/conversion/bailingmoe3.py index 20bba23e51..36b9315649 100644 --- a/conversion/bailingmoe3.py +++ b/conversion/bailingmoe3.py @@ -9,7 +9,9 @@ import torch if TYPE_CHECKING: from torch import Tensor -from .base import ModelBase, TextModel, gguf +from .base import ModelBase, MmprojModel, TextModel, gguf + +from .qwen3vl import Qwen3VLVisionModel @ModelBase.register("BailingMoeV3ForCausalLM") @@ -74,7 +76,7 @@ class BailingMoeV3Model(TextModel): self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"]) self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["moe_shared_expert_intermediate_size"]) - self.gguf_writer.add_expert_shared_count(self.hparams["num_shared_experts"]) + self.gguf_writer.add_expert_shared_count(self.hparams.get("num_shared_experts", 1)) self.gguf_writer.add_leading_dense_block_count(self.hparams["first_k_dense_replace"]) self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"]) self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"]) @@ -191,3 +193,111 @@ class BailingMoeV3Model(TextModel): experts = [name for layer in self._experts for name in layer] if experts: raise ValueError(f"Unprocessed experts: {experts}") + + +@ModelBase.register("BailingMoeV3VLForConditionalGeneration") +@ModelBase.example("inclusionAI/Ling-3.0-flash-VL") +class BailingMoeV3VLModel(BailingMoeV3Model): + model_arch = gguf.MODEL_ARCH.BAILINGMOE3 + + def index_tensors(self, remote_hf_model_id: str | None = None): + # hoist text_config before the shared BailingMoeV3 logic runs: + # ModelBase.__init__ calls this with the raw VL config, where the text + # dims still live under text_config + if "text_config" in self.hparams: + self.hparams = {**self.hparams, **self.hparams["text_config"]} + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + mrope_section = self.hparams.get("mrope_section") + if mrope_section is None: + raise ValueError("BailingMoeV3VL requires mrope_section in the config") + if sum(mrope_section[:3]) * 2 != self.hparams["qk_rope_head_dim"]: + raise ValueError( + f"mrope_section {mrope_section[:3]} counts rope pairs and must sum to" + f" qk_rope_head_dim / 2 = {self.hparams['qk_rope_head_dim'] // 2}" + ) + # mrope_section is [t, h, w]; pad to the 4-wide sections array + self.gguf_writer.add_rope_dimension_sections(list(mrope_section[:3]) + [0]) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + # Skip projector tensors; the vision tower is skipped by TextModel.filter_tensors + if name.startswith("linear_proj"): + return None + + return super().filter_tensors(item) + + +@ModelBase.register("BailingMoeV3VLForConditionalGeneration") +@ModelBase.example("inclusionAI/Ling-3.0-flash-VL") +class BailingMoeV3VLVisionModel(Qwen3VLVisionModel): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + assert self.hparams_vision is not None + + if self.hparams_vision.get("disable_merger_proj") is not True: + raise ValueError("BailingMoeV3VL requires disable_merger_proj=true") + + # out_hidden_size is the vision encoder output (post spatial merge, pre linear_proj) + self.image_emb_dim = self.hparams_vision.get("out_hidden_size") + if self.image_emb_dim is None: + raise ValueError("BailingMoeV3VL vision config requires out_hidden_size") + + def set_gguf_parameters(self): + assert self.hparams_vision is not None + MmprojModel.set_gguf_parameters(self) # skip Qwen3VLVisionModel parameters + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.LING3VL) + self.gguf_writer.add_vision_use_gelu(True) + + merge_size = self.hparams_vision.get("spatial_merge_size") + if merge_size is not None: + self.gguf_writer.add_vision_spatial_merge_size(int(merge_size)) + + rms_norm_eps = self.global_config.get("text_config", {}).get("rms_norm_eps", 1e-6) + self.gguf_writer.add_vision_attention_layernorm_eps(rms_norm_eps) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + if name.startswith("lm_head."): + return None + + if name.startswith("linear_proj"): + # top-level projector MLP: linear_proj.0 -> mm.0, linear_proj.2 -> mm.2 + parts = name.split(".") + if len(parts) != 3: + raise ValueError(f"Unexpected linear_proj tensor: {name}") + idx, suffix = int(parts[1]), parts[2] + name = f"mm.{idx}.{suffix}" + # the qwen3vl filter keeps only visual.*; skip it for the renamed projector tensors + return MmprojModel.filter_tensors((name, gen)) + + if name.startswith("model.visual."): + name = name.replace("model.visual.", "visual.", 1) + + if not name.startswith("visual."): + return None + + return super().filter_tensors((name, gen)) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + assert self.hparams_vision is not None + + if name.startswith("mm.0.") or name.startswith("mm.2."): + # top-level projector MLP (linear_proj.0 / linear_proj.2, renamed by filter_tensors) + yield (name, data_torch) + return + + if name == "visual.merger.norm.weight" or name == "visual.merger.norm.bias": + # the merger is norm-only for Ling: per-patch LayerNorm before the spatial merge + new_name = f"mm.input_norm.{name.split('.')[-1]}" + yield (new_name, data_torch) + return + + # Ling has no patch bias; the Conv3D split below matches the stock qwen3vl path + yield from Qwen3VLVisionModel.modify_tensors(self, data_torch, name, bid) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 27c83516ec..f585adcee3 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -650,6 +650,7 @@ class VISION_PROJECTOR_TYPE(IntEnum): GEMMA3N = auto() GEMMA3 = auto() QWEN3VL = auto() + LING3VL = auto() STEP3VL = auto() COGVLM = auto() @@ -1407,6 +1408,7 @@ VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = { VISION_PROJECTOR_TYPE.MERGER: "qwen2vl_merger", VISION_PROJECTOR_TYPE.GEMMA3: "gemma3", VISION_PROJECTOR_TYPE.QWEN3VL: "qwen3vl_merger", + VISION_PROJECTOR_TYPE.LING3VL: "ling3vl", VISION_PROJECTOR_TYPE.STEP3VL: "step3vl", } @@ -5829,6 +5831,7 @@ class VisionProjectorType: QWEN25VL = "qwen2.5vl_merger" EXAONE4_5 = "exaone4_5" QWEN3VL = "qwen3vl_merger" + LING3VL = "ling3vl" STEP3VL = "step3vl" ULTRAVOX = "ultravox" INTERNVL = "internvl" diff --git a/src/llama-model.cpp b/src/llama-model.cpp index e195f50d0b..3801e5cbe7 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2955,7 +2955,6 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_GRANITE_SWA: case LLM_ARCH_CHAMELEON: case LLM_ARCH_BAILINGMOE: - case LLM_ARCH_BAILINGMOE3: case LLM_ARCH_NEO_BERT: case LLM_ARCH_SMOLLM3: case LLM_ARCH_ARCEE: @@ -2970,6 +2969,10 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_DOTS3NOTE: case LLM_ARCH_NANBEIGE: case LLM_ARCH_POCKETTTS: + return LLAMA_ROPE_TYPE_NORM; + case LLM_ARCH_BAILINGMOE3: + // VL files carry mrope sections; text-only files keep NORM rope + return model->hparams.use_mrope() ? LLAMA_ROPE_TYPE_MROPE : LLAMA_ROPE_TYPE_NORM; // HY_V4 rotates consecutive pairs, matching the reference implementation case LLM_ARCH_HY_V4: return LLAMA_ROPE_TYPE_NORM; diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cpp index e208c7d5aa..907b25c67c 100644 --- a/src/models/bailingmoe3.cpp +++ b/src/models/bailingmoe3.cpp @@ -15,6 +15,7 @@ void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) { hparams.kda_safe_gate = true; } ml.get_key(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound); + ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false); ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); @@ -233,6 +234,10 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph const int64_t d_conv = hparams.ssm_d_conv; const int64_t n_seqs = ubatch.n_seqs; const int64_t n_seq_tokens = ubatch.n_seq_tokens; + + const bool use_mrope = hparams.use_mrope(); + int sections[4]; + std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); const int64_t qk_head_dim = hparams.n_embd_head_k_mla(); const int64_t v_head_dim = hparams.n_embd_head_v_mla(); const int64_t qk_rope_head_dim = hparams.n_rot(); @@ -326,10 +331,17 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph ggml_row_size(kv_all->type, kv_lora_rank + qk_rope_head_dim), ggml_row_size(kv_all->type, kv_lora_rank)); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + if (use_mrope) { + q_pe = ggml_rope_multi(ctx0, q_pe, inp_pos, nullptr, n_rot, sections, rope_type, + n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = ggml_rope_multi(ctx0, k_pe, inp_pos, nullptr, n_rot, sections, rope_type, + n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + } else { + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + } kv = build_norm(kv, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); @@ -482,10 +494,21 @@ llama_model_bailingmoe3::graph_mtp::graph_mtp(const llama_model & model, const l ggml_row_size(kv_all->type, kv_lora_rank + qk_rope_head_dim), ggml_row_size(kv_all->type, kv_lora_rank)); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + const bool use_mrope = hparams.use_mrope(); + int sections[4]; + std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); + + if (use_mrope) { + q_pe = ggml_rope_multi(ctx0, q_pe, inp_pos, nullptr, n_rot, sections, rope_type, + n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = ggml_rope_multi(ctx0, k_pe, inp_pos, nullptr, n_rot, sections, rope_type, + n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + } else { + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + } kv = build_norm(kv, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 27d00b0dfc..5a8a196c1f 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -333,7 +333,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4)); ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1)); - ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4})); + // mrope sections count rope pairs; Ling 3.0 VL files carry [t, h, w] sections + // summing to n_rot / 2 (n_rot is 64 in this fixture) + if (arch == LLM_ARCH_BAILINGMOE3) { + ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector({8, 12, 12, 0})); + } else { + ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4})); + } if (arch == LLM_ARCH_HY_V4) { ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4)); diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 907468e87e..91db32c402 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -55,6 +55,7 @@ add_library(mtmd models/qwen2vl.cpp models/minimax-m3.cpp models/qwen3vl.cpp + models/ling3vl.cpp models/mimovl.cpp models/qwen3a.cpp models/mimo-audio.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 72148a4d9a..d18c76bf8f 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -450,6 +450,7 @@ enum projector_type { PROJECTOR_TYPE_GLM_EDGE, PROJECTOR_TYPE_QWEN2VL, PROJECTOR_TYPE_QWEN3VL, + PROJECTOR_TYPE_LING3VL, PROJECTOR_TYPE_STEP3VL, PROJECTOR_TYPE_GEMMA3, PROJECTOR_TYPE_GEMMA3NV, @@ -516,6 +517,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_QWEN2VL, "qwen2vl_merger"}, { PROJECTOR_TYPE_QWEN25VL, "qwen2.5vl_merger"}, { PROJECTOR_TYPE_QWEN3VL, "qwen3vl_merger"}, + { PROJECTOR_TYPE_LING3VL, "ling3vl"}, { PROJECTOR_TYPE_STEP3VL, "step3vl"}, { PROJECTOR_TYPE_GEMMA3, "gemma3"}, { PROJECTOR_TYPE_GEMMA3NV, "gemma3nv"}, diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index feceb7ff76..62d146e1d9 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -976,6 +976,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_LING3VL: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_EXAONE4_5: { builder = std::make_unique(ctx, img); @@ -1661,6 +1665,7 @@ struct clip_model_loader { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_LING3VL: { hparams.n_merge = 2; // default value for Qwen 2 and 2.5 hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; @@ -2488,6 +2493,15 @@ struct clip_model_loader { model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); } break; + case PROJECTOR_TYPE_LING3VL: + { + model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); // merger.norm + model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); // merger.norm + model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); // linear_proj.0 + model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); // linear_proj.2 + model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); + } break; case PROJECTOR_TYPE_MIMOVL: { model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); @@ -4049,6 +4063,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_MIMOVL: case PROJECTOR_TYPE_GLM4V: @@ -4075,6 +4090,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_MIMOVL: case PROJECTOR_TYPE_GLM4V: @@ -4155,6 +4171,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_MIMOVL: case PROJECTOR_TYPE_MINIMAX_M3: @@ -4795,6 +4812,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_GLM4V: { const int merge_ratio = hparams.n_merge; @@ -5959,6 +5977,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_QWEN3VL: // main path + deepstack paths return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers); + case PROJECTOR_TYPE_LING3VL: + return ctx->model.mm_1_b->ne[0]; case PROJECTOR_TYPE_MIMOVL: return ctx->model.mm_1_w->ne[1]; case PROJECTOR_TYPE_STEP3VL: @@ -6052,6 +6072,7 @@ int clip_model_n_temporal_merge(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_LING3VL: return 2; default: return 1; diff --git a/tools/mtmd/models/ling3vl.cpp b/tools/mtmd/models/ling3vl.cpp new file mode 100644 index 0000000000..ef8eb3a839 --- /dev/null +++ b/tools/mtmd/models/ling3vl.cpp @@ -0,0 +1,86 @@ +#include "models.h" + +ggml_cgraph * clip_graph_ling3vl::build() { + // same vision tower as qwen3vl, but the merger is norm-only (no fc1/fc2) and + // the projector MLP lives at the top level (mm.0 / mm.2) + GGML_ASSERT(model.class_embedding == nullptr); + GGML_ASSERT(model.mm_input_norm_w != nullptr); // merger norm (pre spatial merge) + + const int batch_size = 1; + const int n_pos = n_patches; + + norm_type norm_t = NORM_TYPE_NORMAL; + + // vision M-RoPE, same layout as qwen3vl: [row, col, row, col] quarters + int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4}; + + ggml_tensor * inp = build_inp_with_temporal_merge(); + + // spatial merge + { + inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w, h, c, b] -> [c, w, h, b] + inp = ggml_cont_4d( + ctx0, inp, + n_embd * 2, n_patches_x / 2, n_patches_y, batch_size); + inp = ggml_reshape_4d( + ctx0, inp, + n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2)); + inp = ggml_permute(ctx0, inp, 0, 2, 1, 3); + inp = ggml_cont_3d( + ctx0, inp, + n_embd, n_patches_x * n_patches_y, batch_size); + } + + // add patch bias + if (model.patch_bias != nullptr) { + inp = ggml_add(ctx0, inp, model.patch_bias); + cb(inp, "patch_bias", -1); + } + + // calculate absolute position embedding and apply + ggml_tensor * learned_pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS); + learned_pos_embd = ggml_cont_4d( + ctx0, learned_pos_embd, + n_embd * 2, n_patches_x / 2, n_patches_y, batch_size); + learned_pos_embd = ggml_reshape_4d( + ctx0, learned_pos_embd, + n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2)); + learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3); + learned_pos_embd = ggml_cont_3d( + ctx0, learned_pos_embd, + n_embd, n_patches_x * n_patches_y, batch_size); + + const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position + ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids); + ggml_set_name(positions, "positions"); + ggml_set_input(positions); + + ggml_tensor * inpL = build_vit( + inp, n_pos, norm_t, hparams.ffn_op, learned_pos_embd, + [&](ggml_tensor * c, const clip_layer &) { + return ggml_rope_multi( + ctx0, c, positions, nullptr, + d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1); + }); + + // multimodal projection (linear_proj MLP over the merged patches) + ggml_tensor * embeddings = inpL; + + // per-patch merger norm, applied post-blocks before the 2x2 merge + // (merger.norm, LayerNorm over n_embd) + embeddings = build_norm(embeddings, model.mm_input_norm_w, model.mm_input_norm_b, norm_t, eps, -1); + cb(embeddings, "merger_norm", -1); + + embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size); + + embeddings = build_ffn(embeddings, + model.mm_0_w, model.mm_0_b, + nullptr, nullptr, + model.mm_1_w, model.mm_1_b, + ffn_op_type::FFN_GELU, -1); + + // build the graph + ggml_build_forward_expand(gf, embeddings); + + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 5945c6d92c..adb5ede309 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -50,6 +50,11 @@ struct clip_graph_qwen3vl : clip_graph_qwen2vl { ggml_cgraph * build() override; }; +struct clip_graph_ling3vl : clip_graph_qwen3vl { + clip_graph_ling3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen3vl(ctx, img) {} + ggml_cgraph * build() override; +}; + struct clip_graph_minimax_m3 : clip_graph { clip_graph_minimax_m3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 00ecadcf4d..2efa4d65d3 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -694,6 +694,7 @@ struct mtmd_context { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_MIMOVL: { // <|vision_start|> ... (image embeddings) ... <|vision_end|>