From e7d9c7424f66d8e400e1e67d12897f3372a50bf3 Mon Sep 17 00:00:00 2001 From: ZX-ModelCloud Date: Fri, 21 Aug 2026 19:00:37 +0800 Subject: [PATCH] feat: add DeepSeek V3.2 defusion support --- README.md | 2 +- defuser/model_registry.py | 3 +++ pyproject.toml | 2 +- tests/test_candidate_coverage.py | 17 +++++++++++++++++ tests/test_meta_model_defusion.py | 15 +++++++++++++++ 5 files changed, 37 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7a1fecb..795178c 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Defuser currently supports the following `transformers>=5.3.0` `model_type` valu | Pattern | Supported model types | Defused op performed ⚙️ | | --- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --- | | Standard routed expert tensors 🧱 | `deepseek_v2`, `dots1`, `ernie4_5_moe`, `ernie4_5_vl_moe`, `exaone_moe`, `flex_olmo`, `glm4_moe_lite`, `glm4v_moe`, `hunyuan_v1_moe`, `jamba`, `laguna`, `lfm2_moe`, `minimax`, `minimax_m2`, `olmoe`, `qwen3_vl_moe`, `solar_open`, `solar_open2` | Splits fused expert tensors or registered expert buffers into numbered expert `nn.Linear` modules with per-expert `gate_proj`, `up_proj`, and `down_proj`. | -| Mixed sparse and shared experts | `deepseek_v3`, deepseek_v4`, `glm_moe_dsa`, `qwen3_5_moe`, `qwen3_5_moe_text` | Runtime expert tensor defusion for routed experts while preserving the model's shared-expert path. | +| Mixed sparse and shared experts | `deepseek_v3`, `deepseek_v32`, `deepseek_v4`, `glm_moe_dsa`, `qwen3_5_moe`, `qwen3_5_moe_text` | Runtime expert tensor defusion for routed experts while preserving the model's shared-expert path. | | Transposed or packed expert tensors | `gpt_oss`, `phimoe` | Splits transposed fused expert `gate_up_proj` tensors into per-expert `gate_proj` + `up_proj`, preserves expert bias when present, and converts expert tensors into numbered expert `nn.Linear` modules. | | Flattened expert layout | `dbrx` | Rebuilds the flattened DBRX expert FFN weights into numbered expert `gate_proj`, `up_proj`, and `down_proj` `nn.Linear` modules. | | Batched expert-input execution | `llama4` | Runtime expert tensor defusion plus preservation of the llama4 batched expert-input execution contract. | diff --git a/defuser/model_registry.py b/defuser/model_registry.py index c0d6e07..a9d14a0 100644 --- a/defuser/model_registry.py +++ b/defuser/model_registry.py @@ -37,6 +37,9 @@ class PATCH(str, Enum): "deepseek_v3": { "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, }, + "deepseek_v32": { + "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, + }, "deepseek_v4": { "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, }, diff --git a/pyproject.toml b/pyproject.toml index f596261..80ae686 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "Defuser" -version = "0.0.25" +version = "0.0.26" description = "Model defuser helper for HF Transformers." readme = "README.md" requires-python = ">=3.9" diff --git a/tests/test_candidate_coverage.py b/tests/test_candidate_coverage.py index d3048c4..5208ddc 100644 --- a/tests/test_candidate_coverage.py +++ b/tests/test_candidate_coverage.py @@ -161,6 +161,22 @@ def _standard_hidden(case: dict) -> torch.Tensor: "route_indices": [[0], [1], [2], [3], [0]], "expert_attrs": ("gate_proj", "up_proj", "down_proj"), }, + { + "model_type": "deepseek_v32", + "module_path": "transformers.models.deepseek_v32.modeling_deepseek_v32", + "class_name": "DeepseekV32Experts", + "config_module": "transformers.models.deepseek_v32.configuration_deepseek_v32", + "config_name": "DeepseekV32Config", + "config_updates": { + "hidden_size": 64, + "moe_intermediate_size": 32, + "n_routed_experts": 4, + "hidden_act": "silu", + }, + "input_dim": 64, + "route_indices": [[0], [1], [2], [3], [0]], + "expert_attrs": ("gate_proj", "up_proj", "down_proj"), + }, { "model_type": "dots1", "module_path": "transformers.models.dots1.modeling_dots1", @@ -547,6 +563,7 @@ def _standard_hidden(case: dict) -> torch.Tensor: "dbrx", "deepseek_v2", "deepseek_v3", + "deepseek_v32", "dia", "dots1", "ernie4_5_moe", diff --git a/tests/test_meta_model_defusion.py b/tests/test_meta_model_defusion.py index 20d97e4..cc942f2 100644 --- a/tests/test_meta_model_defusion.py +++ b/tests/test_meta_model_defusion.py @@ -155,6 +155,8 @@ def _build_model_config(case: dict): config.ffn_config.ffn_act_fn = {"name": "silu"} elif model_type == "deepseek_v3": config.first_k_dense_replace = 0 + elif model_type == "deepseek_v32": + config.mlp_layer_types = ["sparse"] * config.num_hidden_layers elif model_type == "ernie4_5_vl_moe": config.text_config.moe_intermediate_size = [32, 32] config.text_config.rope_parameters = { @@ -342,6 +344,19 @@ def _validate_defused_module(case: dict, module) -> None: "validator": "experts", "min_targets": 2, }, + { + "model_type": "deepseek_v32", + "mode": "convert", + "model_module": "transformers.models.deepseek_v32.modeling_deepseek_v32", + "model_class": "DeepseekV32ForCausalLM", + "config_module": "transformers.models.deepseek_v32.configuration_deepseek_v32", + "config_class": "DeepseekV32Config", + "target_class_paths": ( + "transformers.models.deepseek_v32.modeling_deepseek_v32.DeepseekV32Experts", + ), + "validator": "experts", + "min_targets": 2, + }, { "model_type": "deepseek_v4", "mode": "convert",