[model] add glm4moe (#8689)

2025-07-25 19:53:45 +08:00
parent c6e2871944
commit d6767f355a
3 changed files with 65 additions and 0 deletions
--- a/src/llamafactory/model/model_utils/moe.py
+++ b/src/llamafactory/model/model_utils/moe.py
@@ -57,6 +57,11 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None:

        _set_z3_leaf_modules(model, [GraniteMoeMoE])

+    if model_type == "glm4_moe":
+        from transformers.models.glm4_moe.modeling_glm4_moe import Glm4MoeMoE
+
+        _set_z3_leaf_modules(model, [Glm4MoeMoE])
+
    if model_type == "jamba":
        from transformers.models.jamba.modeling_jamba import JambaSparseMoeBlock