From e66e94f032aa8f71fb7dfea744fa33000add2d70 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 1 Sep 2026 03:14:36 +0200 Subject: [PATCH] use GGML_ROPE_TYPE_VISION --- conversion/deepseek.py | 15 --------------- tools/mtmd/clip.cpp | 15 +++++---------- tools/mtmd/models/deepseek4v.cpp | 17 +++++++---------- 3 files changed, 12 insertions(+), 35 deletions(-) diff --git a/conversion/deepseek.py b/conversion/deepseek.py index 9344759130..14ef665c43 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -1069,16 +1069,6 @@ class DeepseekV4FlashVisionModel(MmprojModel): assert self.global_config["vision_max_n_token"] == 384 assert self.global_config["vision_max_wh_ratio"] == 8 - @staticmethod - def _permute_rope(w: Tensor, n_head: int) -> Tensor: - # reference 2D RoPE rotates pairs (dim j, dim j+32): dims [0,16) by row pos, [16,32) by col pos (see apply_rotary in inference/vision.py) - # permute to the build_rope_2d layout: dims [0,32) = adjacent row pairs, dims [32,64) = adjacent col pairs - out_dim = w.shape[0] - head_dim = out_dim // n_head - w = w.reshape(n_head, 2, 2, head_dim // 4, *w.shape[1:]) - w = w.movedim(1, 3) # (head, a, b, f, ...) -> (head, b, f, a, ...) - return w.reshape(out_dim, *w.shape[4:]) - @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, _ = item @@ -1087,9 +1077,6 @@ class DeepseekV4FlashVisionModel(MmprojModel): return super().filter_tensors(item) def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - assert self.hparams_vision is not None - n_head = self.hparams_vision["num_attention_heads"] - if name == "vision.patch_embed.proj.weight": # nn.Linear over flattened (3, p, p) patches == conv2d weight p = self.hparams_vision["patch_size"] @@ -1097,8 +1084,6 @@ class DeepseekV4FlashVisionModel(MmprojModel): if ".attn.wqkv." in name: q, k, v = data_torch.chunk(3, dim=0) - q = self._permute_rope(q, n_head) - k = self._permute_rope(k, n_head) yield from super().modify_tensors(q, name.replace("wqkv", "wq"), bid) yield from super().modify_tensors(k, name.replace("wqkv", "wk"), bid) yield from super().modify_tensors(v, name.replace("wqkv", "wv"), bid) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 4bd74e0b7c..19c6745c13 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -5076,19 +5076,14 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } break; case PROJECTOR_TYPE_DEEPSEEK4V: { - // set the 2D positions + // set the 2D positions (mrope layout, only the first 2 channels are used) int n_patches_per_col = image_size_width / patch_size; - std::vector pos_data(n_pos); - // dimension H + std::vector positions(n_pos * 4, 0); for (int i = 0; i < n_pos; i++) { - pos_data[i] = i / n_patches_per_col; + positions[i] = i / n_patches_per_col; // row + positions[n_pos + i] = i % n_patches_per_col; // col } - set_input_i32("pos_h", pos_data); - // dimension W - for (int i = 0; i < n_pos; i++) { - pos_data[i] = i % n_patches_per_col; - } - set_input_i32("pos_w", pos_data); + set_input_i32("positions", positions); // token block layout index (see clip_graph_deepseek4v::build) // rows [0, n_grid) are the aligner output, the sentinels follow diff --git a/tools/mtmd/models/deepseek4v.cpp b/tools/mtmd/models/deepseek4v.cpp index 577ced167c..d25df9156b 100644 --- a/tools/mtmd/models/deepseek4v.cpp +++ b/tools/mtmd/models/deepseek4v.cpp @@ -19,18 +19,15 @@ ggml_cgraph * clip_graph_deepseek4v::build() { const int n_merge = hparams.n_merge; // 2D input positions - ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); - ggml_set_name(pos_h, "pos_h"); - ggml_set_input(pos_h); - - ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); - ggml_set_name(pos_w, "pos_w"); - ggml_set_input(pos_w); + ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4); + ggml_set_name(positions, "positions"); + ggml_set_input(positions); + int sections[4] = {d_head/4, d_head/4, 0, 0}; auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { - // row dims rotate in the first half, col dims in the second half - // the head dims are permuted at conversion time - return build_rope_2d(ctx0, cur, pos_h, pos_w, hparams.rope_theta, false); + return ggml_rope_multi(ctx0, cur, positions, nullptr, + d_head/2, sections, GGML_ROPE_TYPE_VISION, + 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); }; ggml_tensor * inp = build_inp();