use GGML_ROPE_TYPE_VISION

conversion/deepseek.py

@@ -1069,16 +1069,6 @@ class DeepseekV4FlashVisionModel(MmprojModel):
         assert self.global_config["vision_max_n_token"] == 384
         assert self.global_config["vision_max_wh_ratio"] == 8
 
-    @staticmethod
-    def _permute_rope(w: Tensor, n_head: int) -> Tensor:
-        # reference 2D RoPE rotates pairs (dim j, dim j+32): dims [0,16) by row pos, [16,32) by col pos (see apply_rotary in inference/vision.py)
-        # permute to the build_rope_2d layout: dims [0,32) = adjacent row pairs, dims [32,64) = adjacent col pairs
-        out_dim = w.shape[0]
-        head_dim = out_dim // n_head
-        w = w.reshape(n_head, 2, 2, head_dim // 4, *w.shape[1:])
-        w = w.movedim(1, 3) # (head, a, b, f, ...) -> (head, b, f, a, ...)
-        return w.reshape(out_dim, *w.shape[4:])
-
     @classmethod
     def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
         name, _ = item
@@ -1087,9 +1077,6 @@ class DeepseekV4FlashVisionModel(MmprojModel):
         return super().filter_tensors(item)
 
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        assert self.hparams_vision is not None
-        n_head = self.hparams_vision["num_attention_heads"]
-
         if name == "vision.patch_embed.proj.weight":
             # nn.Linear over flattened (3, p, p) patches == conv2d weight
             p = self.hparams_vision["patch_size"]
@@ -1097,8 +1084,6 @@ class DeepseekV4FlashVisionModel(MmprojModel):
 
         if ".attn.wqkv." in name:
             q, k, v = data_torch.chunk(3, dim=0)
-            q = self._permute_rope(q, n_head)
-            k = self._permute_rope(k, n_head)
             yield from super().modify_tensors(q, name.replace("wqkv", "wq"), bid)
             yield from super().modify_tensors(k, name.replace("wqkv", "wk"), bid)
             yield from super().modify_tensors(v, name.replace("wqkv", "wv"), bid)

tools/mtmd/clip.cpp

@@ -5076,19 +5076,14 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
             } break;
         case PROJECTOR_TYPE_DEEPSEEK4V:
             {
-                // set the 2D positions
+                // set the 2D positions (mrope layout, only the first 2 channels are used)
                 int n_patches_per_col = image_size_width / patch_size;
-                std::vector<int> pos_data(n_pos);
-                // dimension H
+                std::vector<int32_t> positions(n_pos * 4, 0);
                 for (int i = 0; i < n_pos; i++) {
-                    pos_data[i] = i / n_patches_per_col;
+                    positions[i]         = i / n_patches_per_col; // row
+                    positions[n_pos + i] = i % n_patches_per_col; // col
                 }
-                set_input_i32("pos_h", pos_data);
-                // dimension W
-                for (int i = 0; i < n_pos; i++) {
-                    pos_data[i] = i % n_patches_per_col;
-                }
-                set_input_i32("pos_w", pos_data);
+                set_input_i32("positions", positions);
 
                 // token block layout index (see clip_graph_deepseek4v::build)
                 // rows [0, n_grid) are the aligner output, the sentinels follow

tools/mtmd/models/deepseek4v.cpp

@@ -19,18 +19,15 @@ ggml_cgraph * clip_graph_deepseek4v::build() {
     const int n_merge = hparams.n_merge;
 
     // 2D input positions
-    ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
-    ggml_set_name(pos_h, "pos_h");
-    ggml_set_input(pos_h);
-
-    ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
-    ggml_set_name(pos_w, "pos_w");
-    ggml_set_input(pos_w);
+    ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4);
+    ggml_set_name(positions, "positions");
+    ggml_set_input(positions);
 
+    int sections[4] = {d_head/4, d_head/4, 0, 0};
     auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
-        // row dims rotate in the first half, col dims in the second half
-        // the head dims are permuted at conversion time
-        return build_rope_2d(ctx0, cur, pos_h, pos_w, hparams.rope_theta, false);
+        return ggml_rope_multi(ctx0, cur, positions, nullptr,
+            d_head/2, sections, GGML_ROPE_TYPE_VISION,
+            0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
     };
 
     ggml_tensor * inp = build_inp();