ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[model] support Internvl3_5 (#9028)

Browse Source
This commit is contained in:
Kingsley
2025-08-28 17:12:00 +08:00
committed by GitHub
parent 1c675522fd
commit 185f0556d4
4 changed files with 29 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/llamafactory/model/model_utils/moe.py
View File

@@ -39,6 +39,9 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None:
return
model_type = getattr(model.config, "model_type", None)
text_config = getattr(model.config, "text_config", None)
text_architectures = getattr(text_config, "architectures", None)
if model_type == "dbrx":
from transformers.models.dbrx.modeling_dbrx import DbrxFFN
@@ -102,7 +105,7 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None:
_set_z3_leaf_modules(model, [Qwen2MoeSparseMoeBlock])
if model_type == "qwen3_moe":
if model_type == "qwen3_moe" or text_architectures == "Qwen3MoeForCausalLM": # for internvl_3_5
from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeSparseMoeBlock
_set_z3_leaf_modules(model, [Qwen3MoeSparseMoeBlock])
@@ -113,6 +116,8 @@ def configure_moe(config: "PretrainedConfig", model_args: "ModelArguments", is_t
return
model_type = getattr(config, "model_type", None)
text_config = getattr(config, "text_config", None) # for multimodal model
if model_type in [
"dbrx",
"granitemoe",
@@ -127,9 +132,18 @@ def configure_moe(config: "PretrainedConfig", model_args: "ModelArguments", is_t
]:
setattr(config, "output_router_logits", True)
if text_config and getattr(text_config, "model_type", None) in [
"glm4v_moe_text", # glmv4_5
"qwen3_moe", # internvl_3_5
]:
setattr(text_config, "output_router_logits", True)
if model_type in ["granitemoe", "jamba", "llama4", "mixtral", "olmoe", "phimoe", "qwen2_moe", "qwen3_moe"]:
setattr(config, "router_aux_loss_coef", model_args.moe_aux_loss_coef)
elif text_config and getattr(text_config, "model_type", None) in ["qwen3_moe"]:
setattr(text_config, "router_aux_loss_coef", model_args.moe_aux_loss_coef)
elif model_type == "deepseek":
setattr(config, "aux_loss_alpha", model_args.moe_aux_loss_coef)