From dac869b0a05d14073d174f330bcd86b9122df549 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Wed, 26 Aug 2026 12:05:31 +0200 Subject: [PATCH] conversion : fix Nemotron 3.5 Lightning layers (#27729) This commit contains a fix for the conversion of NVIDIA Nemotron 3.5 Lightning which currently incorrectly converts when using a transformers version later than 5.5.1. When converting using [convert](https://github.com/ggml-org/convert) the transformers version is 5.13.1 and this produces the following: ```console WARNING:gguf.gguf_writer:Duplicated key name 'nemotron_h_moe.attention.head_count_kv', overwriting it with new value [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] of type ARRAY ``` This does not happen with transformers 5.5.1. The reason seems to be that the configuration is different in later versions, for example when using 5.13.1 the configuration block looks like this: ```console transformers 5.13.1 raw has layers_block_type: True autoconfig has layers_block_type: True autoconfig layers_block_type: [ 'linear_attention', 'moe', 'linear_attention', 'moe', 'linear_attention', 'full_attention', 'moe', ... ] ``` And with 5.5.1 we get: ```console transformers 5.5.1 raw has layers_block_type: True autoconfig has layers_block_type: True autoconfig layers_block_type: [ 'mamba', 'moe', 'mamba', 'moe', 'mamba', 'attention', 'moe' ... ] ``` In our conversion script we only match for attention, not full attention which is causing this issue. With the changes in this commit the output with transformers 5.13.1 will be: ```console (venv) $ gguf-dump models/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16.gguf | grep head_count_kv INFO:gguf-dump:* Loading: models/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16.gguf 29: [INT32] | 52 | nemotron_h_moe.attention.head_count_kv = [0, 0, 0, 0, 0, 2, ...] ``` Resolves: https://github.com/ggml-org/llama.cpp/issues/27718 Refs: https://github.com/ggml-org/convert/actions/runs/32949047680/job/98116096069#step:5:2391 --- conversion/nemotron.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/conversion/nemotron.py b/conversion/nemotron.py index e5d1671851..cd0d48c8f0 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -202,6 +202,10 @@ class NemotronHModel(GraniteHybridModel): is_moe: bool = False supports_mtp_export = True + _SSM_LAYER_TYPES = {"mamba", "linear_attention"} + _ATTN_LAYER_TYPES = {"attention", "full_attention"} + _MLP_LAYER_TYPES = {"moe"} + def __init__(self, *args, **kwargs): # We have to determine the correct model architecture (MoE vs non-MoE) before # calling the parent __init__. This is because the parent constructor @@ -242,8 +246,8 @@ class NemotronHModel(GraniteHybridModel): self._ssm_layers = [i for i, val in enumerate(pattern) if val == "M"] self._mlp_layers = [i for i, val in enumerate(pattern) if val == ("E" if self.is_moe else "-")] else: - self._ssm_layers = [i for i, val in enumerate(pattern) if val == "mamba"] - self._mlp_layers = [i for i, val in enumerate(pattern) if val == "moe"] + self._ssm_layers = [i for i, val in enumerate(pattern) if val in self._SSM_LAYER_TYPES] + self._mlp_layers = [i for i, val in enumerate(pattern) if val in self._MLP_LAYER_TYPES] # `--no-mtp` drops it entirely; `--mtp` exports only the MTP head self._mtp_bid: int | None = None @@ -272,7 +276,7 @@ class NemotronHModel(GraniteHybridModel): if isinstance(pattern, str): return [i for i, val in enumerate(pattern) if val == "*"] - return [i for i, val in enumerate(pattern) if val == "attention"] + return [i for i, val in enumerate(pattern) if val in self._ATTN_LAYER_TYPES] @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: