From b21e4de74567f5eef213765c9476a843c2e43f0d Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Sat, 22 Aug 2026 16:33:47 +0200 Subject: [PATCH] mtmd: use ggml_rope_set_offset (#27521) * mtmd: use ggml_rope_set_offset * add comment --- tools/mtmd/clip-graph.h | 12 ++++-- tools/mtmd/clip.cpp | 66 +++++++++++--------------------- tools/mtmd/models/gemma4v.cpp | 64 +++++++++++-------------------- tools/mtmd/models/minimax-m3.cpp | 24 ++++-------- 4 files changed, 62 insertions(+), 104 deletions(-) diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.h index 2cf1b683a7..bbee35bead 100644 --- a/tools/mtmd/clip-graph.h +++ b/tools/mtmd/clip-graph.h @@ -137,9 +137,15 @@ struct clip_graph { int il, ggml_tensor * sinks = nullptr) const; - // implementation of the 2D RoPE without adding a new op in ggml - // this is not efficient (use double the memory), but works on all backends - // TODO: there was a more efficient which relies on ggml_view and ggml_rope_ext_inplace, but the rope inplace does not work well with non-contiguous tensors ; we should fix that and revert back to the original implementation in https://github.com/ggml-org/llama.cpp/pull/13065 + // implementation of the 2D RoPE using two ggml_rope_ext calls + // + // unlike GGML_ROPE_TYPE_VISION which forces NEOX ordering, this rotates adjacent pairs (normal ordering) + // + // example: + // given a single head with size = 8 --> [00000000] + // dims [0, 4) rotate with pos_a, dims [4, 8) rotate with pos_b --> [aaaabbbb] + // interleave_freq = false --> both halves use the same inv_freq set (like GGML_ROPE_TYPE_VISION) + // interleave_freq = true --> first half uses even inv_freq, second half uses odd inv_freq (used by pixtral) ggml_tensor * build_rope_2d( ggml_context * ctx0, ggml_tensor * cur, diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 9977ed4909..89ca65a7bb 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -819,8 +819,6 @@ ggml_tensor * clip_graph::build_attn( } // implementation of the 2D RoPE without adding a new op in ggml -// this is not efficient (use double the memory), but works on all backends -// TODO: there was a more efficient which relies on ggml_view and ggml_rope_ext_inplace, but the rope inplace does not work well with non-contiguous tensors ; we should fix that and revert back to the original implementation in https://github.com/ggml-org/llama.cpp/pull/13065 ggml_tensor * clip_graph::build_rope_2d( ggml_context * ctx0, ggml_tensor * cur, @@ -829,9 +827,7 @@ ggml_tensor * clip_graph::build_rope_2d( const float freq_base, const bool interleave_freq ) { - const int64_t n_dim = cur->ne[0]; - const int64_t n_head = cur->ne[1]; - const int64_t n_pos = cur->ne[2]; + const int64_t n_dim = cur->ne[0]; // for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos) // we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3 @@ -845,46 +841,30 @@ ggml_tensor * clip_graph::build_rope_2d( ? std::pow(freq_base, (float)-2/n_dim) : 1.0; - // first half - ggml_tensor * first; - { - first = ggml_view_3d(ctx0, cur, - n_dim/2, n_head, n_pos, - cur->nb[1], - cur->nb[2], - 0); - first = ggml_rope_ext( - ctx0, - first, - pos_a, // positions - nullptr, // freq factors - n_dim/2, // n_dims - 0, 0, freq_base, - 1.0f, 0.0f, 1.0f, 0.0f, 0.0f - ); - } + // first half, dims [0, n_dim/2) + cur = ggml_rope_ext( + ctx0, + cur, + pos_a, // positions + nullptr, // freq factors + n_dim/2, // n_dims + 0, 0, freq_base, + 1.0f, 0.0f, 1.0f, 0.0f, 0.0f + ); - // second half - ggml_tensor * second; - { - second = ggml_view_3d(ctx0, cur, - n_dim/2, n_head, n_pos, - cur->nb[1], - cur->nb[2], - n_dim/2 * ggml_element_size(cur)); - second = ggml_rope_ext( - ctx0, - second, - pos_b, // positions - nullptr, // freq factors - n_dim/2, // n_dims - 0, 0, freq_base, - freq_scale_odd, - 0.0f, 1.0f, 0.0f, 0.0f - ); - } + // second half, dims [n_dim/2, n_dim) + cur = ggml_rope_ext( + ctx0, + cur, + pos_b, // positions + nullptr, // freq factors + n_dim/2, // n_dims + 0, 0, freq_base, + freq_scale_odd, + 0.0f, 1.0f, 0.0f, 0.0f + ); + cur = ggml_rope_set_offset(cur, n_dim/2); - cur = ggml_concat(ctx0, first, second, 0); return cur; } diff --git a/tools/mtmd/models/gemma4v.cpp b/tools/mtmd/models/gemma4v.cpp index 87cbd43fc5..4484389470 100644 --- a/tools/mtmd/models/gemma4v.cpp +++ b/tools/mtmd/models/gemma4v.cpp @@ -44,51 +44,31 @@ ggml_cgraph * clip_graph_gemma4v::build() { // similar to build_rope_2d, but use neox ordering auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { - const int64_t n_dim = cur->ne[0]; - const int64_t n_head = cur->ne[1]; - const int64_t n_pos = cur->ne[2]; + const int64_t n_dim = cur->ne[0]; - // first half - ggml_tensor * first; - { - first = ggml_view_4d(ctx0, cur, - n_dim/2, n_head, n_pos, n_batch, - cur->nb[1], - cur->nb[2], - cur->nb[3], - 0); - first = ggml_rope_ext( - ctx0, - first, - pos_x, // positions - nullptr, // freq factors - n_dim/2, // n_dims - GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, - 1.0f, 0.0f, 1.0f, 0.0f, 0.0f - ); - } + // first half, dims [0, n_dim/2) + cur = ggml_rope_ext( + ctx0, + cur, + pos_x, // positions + nullptr, // freq factors + n_dim/2, // n_dims + GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, + 1.0f, 0.0f, 1.0f, 0.0f, 0.0f + ); - // second half - ggml_tensor * second; - { - second = ggml_view_4d(ctx0, cur, - n_dim/2, n_head, n_pos, n_batch, - cur->nb[1], - cur->nb[2], - cur->nb[3], - n_dim/2 * ggml_element_size(cur)); - second = ggml_rope_ext( - ctx0, - second, - pos_y, // positions - nullptr, // freq factors - n_dim/2, // n_dims - GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, - 1.0f, 0.0f, 1.0f, 0.0f, 0.0f - ); - } + // second half, dims [n_dim/2, n_dim) + cur = ggml_rope_ext( + ctx0, + cur, + pos_y, // positions + nullptr, // freq factors + n_dim/2, // n_dims + GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, + 1.0f, 0.0f, 1.0f, 0.0f, 0.0f + ); + cur = ggml_rope_set_offset(cur, n_dim/2); - cur = ggml_concat(ctx0, first, second, 0); return cur; }; diff --git a/tools/mtmd/models/minimax-m3.cpp b/tools/mtmd/models/minimax-m3.cpp index 447621754e..256e531057 100644 --- a/tools/mtmd/models/minimax-m3.cpp +++ b/tools/mtmd/models/minimax-m3.cpp @@ -2,30 +2,22 @@ ggml_tensor * clip_graph_minimax_m3::apply_rope( ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) { - const int64_t Hn = x->ne[1]; - const int64_t P = x->ne[2]; - const size_t es = ggml_element_size(x); - const int dh = (int) x->ne[0]; - const int axd = 2 * ((2 * (dh / 2) / 3) / 2); + const int dh = (int) x->ne[0]; + const int axd = 2 * ((2 * (dh / 2) / 3) / 2); - GGML_ASSERT(x->nb[0] == es); GGML_ASSERT(3 * axd <= dh); const float th = hparams.rope_theta; // layout of x is [t, h, w, pad] // t is unrotated, h and w are rotated, pad is unrotated - // note: everything from n_dims onward untouched, so w and pad are rotated in one call. - auto sl = [&](int off, int n) { - return ggml_cont(ctx0, ggml_view_3d(ctx0, x, n, Hn, P, x->nb[1], x->nb[2], (size_t) off * es)); - }; - ggml_tensor * t = sl(0, axd); - ggml_tensor * h = sl(axd, axd); - ggml_tensor * w = sl(2 * axd, dh - 2 * axd); // w + pad + x = ggml_rope_ext(ctx0, x, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + x = ggml_rope_set_offset(x, axd); - h = ggml_rope_ext(ctx0, h, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); - w = ggml_rope_ext(ctx0, w, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); - return ggml_concat(ctx0, ggml_concat(ctx0, t, h, 0), w, 0); + x = ggml_rope_ext(ctx0, x, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + x = ggml_rope_set_offset(x, 2 * axd); + + return x; } ggml_cgraph * clip_graph_minimax_m3::build() {