diff --git a/README.md b/README.md index a0af2ae..7a1fecb 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Defuser currently supports the following `transformers>=5.3.0` `model_type` valu | Pattern | Supported model types | Defused op performed ⚙️ | | --- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --- | -| Standard routed expert tensors 🧱 | `deepseek_v2`, `dots1`, `ernie4_5_moe`, `ernie4_5_vl_moe`, `exaone_moe`, `flex_olmo`, `glm4_moe_lite`, `glm4v_moe`, `hunyuan_v1_moe`, `jamba`, `laguna`, `lfm2_moe`, `minimax`, `minimax_m2`, `olmoe`, `qwen3_vl_moe`, `solar_open` | Splits fused expert tensors or registered expert buffers into numbered expert `nn.Linear` modules with per-expert `gate_proj`, `up_proj`, and `down_proj`. | +| Standard routed expert tensors 🧱 | `deepseek_v2`, `dots1`, `ernie4_5_moe`, `ernie4_5_vl_moe`, `exaone_moe`, `flex_olmo`, `glm4_moe_lite`, `glm4v_moe`, `hunyuan_v1_moe`, `jamba`, `laguna`, `lfm2_moe`, `minimax`, `minimax_m2`, `olmoe`, `qwen3_vl_moe`, `solar_open`, `solar_open2` | Splits fused expert tensors or registered expert buffers into numbered expert `nn.Linear` modules with per-expert `gate_proj`, `up_proj`, and `down_proj`. | | Mixed sparse and shared experts | `deepseek_v3`, deepseek_v4`, `glm_moe_dsa`, `qwen3_5_moe`, `qwen3_5_moe_text` | Runtime expert tensor defusion for routed experts while preserving the model's shared-expert path. | | Transposed or packed expert tensors | `gpt_oss`, `phimoe` | Splits transposed fused expert `gate_up_proj` tensors into per-expert `gate_proj` + `up_proj`, preserves expert bias when present, and converts expert tensors into numbered expert `nn.Linear` modules. | | Flattened expert layout | `dbrx` | Rebuilds the flattened DBRX expert FFN weights into numbered expert `gate_proj`, `up_proj`, and `down_proj` `nn.Linear` modules. | diff --git a/defuser/model_registry.py b/defuser/model_registry.py index 557299c..c468bb1 100644 --- a/defuser/model_registry.py +++ b/defuser/model_registry.py @@ -25,6 +25,12 @@ class PATCH(str, Enum): "deepseek_v2": { "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, }, + "deepseek_ocr2": { + "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, + }, + "inkling_mm_model": { + "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, + }, "deepseek_v3": { "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, }, @@ -268,6 +274,9 @@ class PATCH(str, Enum): "solar_open": { "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, }, + "solar_open2": { + "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, + }, "zamba2": { "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION, }, diff --git a/pyproject.toml b/pyproject.toml index 6817940..4a7fc5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "Defuser" -version = "0.0.23" +version = "0.0.24" description = "Model defuser helper for HF Transformers." readme = "README.md" requires-python = ">=3.9" diff --git a/tests/test_candidate_coverage.py b/tests/test_candidate_coverage.py index bec021c..d3048c4 100644 --- a/tests/test_candidate_coverage.py +++ b/tests/test_candidate_coverage.py @@ -351,6 +351,22 @@ def _standard_hidden(case: dict) -> torch.Tensor: "route_indices": [[0], [1], [2], [3], [0]], "expert_attrs": ("gate_proj", "up_proj", "down_proj"), }, + { + "model_type": "solar_open2", + "module_path": "transformers.models.solar_open2.modeling_solar_open2", + "class_name": "SolarOpen2Experts", + "config_module": "transformers.models.solar_open2.configuration_solar_open2", + "config_name": "SolarOpen2Config", + "config_updates": { + "hidden_size": 64, + "moe_intermediate_size": 32, + "n_routed_experts": 4, + "hidden_act": "silu", + }, + "input_dim": 64, + "route_indices": [[0], [1], [2], [3], [0]], + "expert_attrs": ("gate_proj", "up_proj", "down_proj"), + }, { "model_type": "dbrx", "module_path": "transformers.models.dbrx.modeling_dbrx", @@ -571,6 +587,7 @@ def _standard_hidden(case: dict) -> torch.Tensor: "qwen3_omni_moe", "qwen3_vl_moe", "solar_open", + "solar_open2", "zamba2", } diff --git a/tests/test_meta_model_defusion.py b/tests/test_meta_model_defusion.py index 1acd8b7..d543f77 100644 --- a/tests/test_meta_model_defusion.py +++ b/tests/test_meta_model_defusion.py @@ -779,6 +779,17 @@ def _validate_defused_module(case: dict, module) -> None: "target_class_paths": ("transformers.models.solar_open.modeling_solar_open.SolarOpenNaiveMoe",), "validator": "experts", }, + { + "model_type": "solar_open2", + "mode": "convert", + "model_module": "transformers.models.solar_open2.modeling_solar_open2", + "model_class": "SolarOpen2ForCausalLM", + "config_module": "transformers.models.solar_open2.configuration_solar_open2", + "config_class": "SolarOpen2Config", + "target_class_paths": ("transformers.models.solar_open2.modeling_solar_open2.SolarOpen2Experts",), + "validator": "experts", + "min_targets": 2, + }, { "model_type": "zamba2", "mode": "convert",