From 402a11ee853596eef13bf2ac0266b287c2bb5396 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 1 Sep 2026 11:50:33 +0200 Subject: [PATCH] model: correctly support input vision for deepseek4 --- conversion/deepseek.py | 8 ++++++-- gguf-py/gguf/constants.py | 3 +++ src/llama-arch.cpp | 2 ++ src/llama-arch.h | 1 + src/llama-hparams.h | 3 +++ src/llama-kv-cache.cpp | 4 +++- src/llama-model.h | 1 + src/models/deepseek4.cpp | 14 +++++++++++++- tools/mtmd/mtmd.cpp | 1 + 9 files changed, 33 insertions(+), 4 deletions(-) diff --git a/conversion/deepseek.py b/conversion/deepseek.py index c244e94ec3..abaed7880b 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -578,8 +578,7 @@ class DeepseekV4Model(TextModel): @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item - if (name.startswith(("aligner.", "image_")) - or name.endswith(".ffn.gate.bias_vl")): + if name.startswith(("aligner.", "image_")): return None if name.startswith("mtp."): if not cls.mtp_only: @@ -856,6 +855,7 @@ class DeepseekV4Model(TextModel): "ffn_norm.weight": (gguf.MODEL_TENSOR.FFN_NORM, ".weight"), "ffn.gate.weight": (gguf.MODEL_TENSOR.FFN_GATE_INP, ".weight"), "ffn.gate.bias": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B, ".bias"), + "ffn.gate.bias_vl": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B_VL, ".bias"), "ffn.gate.tid2eid": (gguf.MODEL_TENSOR.FFN_GATE_TID2EID, ".weight"), "ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"), "ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"), @@ -881,6 +881,10 @@ class DeepseekV4Model(TextModel): if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name): return [] + # hash layers route text tokens via tid2eid and image tokens via bias_vl; gate.bias is unused + if name.endswith(".ffn.gate.bias") and bid is not None and bid < self.hparams["num_hash_layers"]: + return [] + tensor_key, suffix = self._map_dsv4_tensor_name(name, bid) if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID: return [] diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 56477c1988..b85f62a311 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -697,6 +697,7 @@ class MODEL_TENSOR(IntEnum): FFN_DOWN_CHEXP = auto() FFN_UP_CHEXP = auto() FFN_EXP_PROBS_B = auto() + FFN_EXP_PROBS_B_VL = auto() # deepseek4 vision (bias for image tokens) FFN_GATE_TID2EID = auto() MOE_LATENT_DOWN = auto() # nemotron 3 super MOE_LATENT_UP = auto() # nemotron 3 super @@ -1449,6 +1450,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up_exps", MODEL_TENSOR.FFN_GATE_UP_EXP: "blk.{bid}.ffn_gate_up_exps", MODEL_TENSOR.FFN_EXP_PROBS_B: "blk.{bid}.exp_probs_b", + MODEL_TENSOR.FFN_EXP_PROBS_B_VL: "blk.{bid}.exp_probs_b_vl", MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid", MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 super MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super @@ -3839,6 +3841,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_GATE_INP, MODEL_TENSOR.FFN_GATE_TID2EID, MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_EXP_PROBS_B_VL, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE_EXP, MODEL_TENSOR.FFN_DOWN_EXP, diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 7adb87411a..446de4ae25 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -457,6 +457,7 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_FFN_UP_SHEXP, "blk.%d.ffn_up_shexp" }, { LLM_TENSOR_FFN_DOWN_SHEXP, "blk.%d.ffn_down_shexp" }, { LLM_TENSOR_FFN_EXP_PROBS_B, "blk.%d.exp_probs_b" }, + { LLM_TENSOR_FFN_EXP_PROBS_B_VL, "blk.%d.exp_probs_b_vl" }, { LLM_TENSOR_FFN_LATENT_DOWN, "blk.%d.ffn_latent_down" }, { LLM_TENSOR_FFN_LATENT_UP, "blk.%d.ffn_latent_up" }, { LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" }, @@ -896,6 +897,7 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_FFN_GATE_CHEXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}}, {LLM_TENSOR_FFN_UP_CHEXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}}, {LLM_TENSOR_FFN_EXP_PROBS_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_FFN_EXP_PROBS_B_VL, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, // altup / laurel (gemma 3n) {LLM_TENSOR_PER_LAYER_TOKEN_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, {LLM_TENSOR_PER_LAYER_MODEL_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, diff --git a/src/llama-arch.h b/src/llama-arch.h index ca7d55a5fd..0c0b994836 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -477,6 +477,7 @@ enum llm_tensor { LLM_TENSOR_FFN_GATE_CHEXPS, LLM_TENSOR_FFN_UP_CHEXPS, LLM_TENSOR_FFN_EXP_PROBS_B, + LLM_TENSOR_FFN_EXP_PROBS_B_VL, LLM_TENSOR_FFN_LATENT_DOWN, LLM_TENSOR_FFN_LATENT_UP, LLM_TENSOR_ATTN_Q_NORM, diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 1411692a89..d43747543d 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -161,6 +161,9 @@ struct llama_hparams { // the size of the sliding window (0 - no SWA) uint32_t n_swa = 0; + // deepseek4 vision: when set to non-causal attention, SWA should not be applied (case: input is multimodal) + bool swa_full_non_causal = false; + // if is_swa_impl[il] == 1, then layer il is SWA // if is_swa_impl[il] == 0, then layer il is dense (i.e. non-SWA) // by default, all layers are dense diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 3e4a4d56f9..f22054c3d6 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1681,7 +1681,9 @@ static void set_input_kq_mask_impl(const args_set_input_kq_mask & args, T * data // apply SWA if any if (swa) { - if (llama_hparams::is_masked_swa(n_swa, swa_type, p0, p1)) { + // see llama_hparams::swa_full_non_causal + const bool in_span = !causal && args.hparams.swa_full_non_causal && p0 >= seq_pos_min[seq_id]; + if (!in_span && llama_hparams::is_masked_swa(n_swa, swa_type, p0, p1)) { goto skip; } } diff --git a/src/llama-model.h b/src/llama-model.h index 38066538ed..ee6bb5ac37 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -362,6 +362,7 @@ struct llama_layer { struct ggml_tensor * ffn_up_b = nullptr; // b3 struct ggml_tensor * ffn_act = nullptr; struct ggml_tensor * ffn_exp_probs_b = nullptr; + struct ggml_tensor * ffn_exp_probs_b_vl = nullptr; // deepseek4 vision (bias for image tokens) struct ggml_tensor * ffn_gate_tid2eid = nullptr; struct ggml_tensor * dflash_attn_conv_base = nullptr; diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 222f222492..680516b230 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -66,6 +66,9 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) { } hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.set_swa_pattern(0); + // tokens of an image span attend bidirectionally to the whole span, the window only applies to older tokens + // ref: get_window_topk_idxs_visible in the reference impl + hparams.swa_full_non_causal = true; for (uint32_t il = hparams.n_layer(); il < hparams.n_layer_all; ++il) { hparams.is_swa_impl[il] = true; } @@ -156,6 +159,8 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) { } else { layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, flags); } + // vision variant only: routing bias for image tokens + layer.ffn_exp_probs_b_vl = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B_VL, "bias", i), {n_expert}, flags | TENSOR_NOT_REQUIRED); layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags); layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, flags); @@ -1275,7 +1280,14 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p const auto & layer = model.layers[il]; ggml_tensor * selected_experts = nullptr; ggml_tensor * exp_probs_b = layer.ffn_exp_probs_b; - if ((uint32_t) il < hparams.dsv4_hash_layer_count) { + + // may apply exp_probs_b_vl is input is from mtmd + const bool is_media = ubatch.embd != nullptr; + if (is_media) { + if (layer.ffn_exp_probs_b_vl) { + exp_probs_b = layer.ffn_exp_probs_b_vl; + } + } else if ((uint32_t) il < hparams.dsv4_hash_layer_count) { selected_experts = ggml_get_rows(ctx0, layer.ffn_gate_tid2eid, res->t_inp_tokens); exp_probs_b = nullptr; } diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index d2b88b1e46..f39f0974e3 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -2132,6 +2132,7 @@ bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk case PROJECTOR_TYPE_GEMMA3: case PROJECTOR_TYPE_GEMMA4V: case PROJECTOR_TYPE_GEMMA4UV: + case PROJECTOR_TYPE_DEEPSEEK4V: return true; default: return false;