add regex of only tune lm and mm_proj

Former-commit-id: 38d540b3e69bceabafafab524fcfc78aeb05612d

src/llamafactory/model/adapter.py

@@ -10,6 +10,7 @@ from ..extras.logging import get_logger
 from .utils.misc import find_all_linear_modules, find_expanded_modules
 from .utils.quantization import QuantizationMethod
 from .utils.unsloth import get_unsloth_peft_model, load_unsloth_peft_model
+from .utils.visual import filter_vision_tower_linear
 
 
 if TYPE_CHECKING:
@@ -58,6 +59,9 @@ def init_adapter(
         if model_args.visual_inputs and hasattr(model, "vision_tower"):  # freeze vision model
             model.vision_tower.requires_grad_(False)
 
+        if model_args.visual_inputs and hasattr(model, "language_model") and model_args.tune_mm_proj:  # freeze language model if only tune mm_proj
+            model.language_model.requires_grad_(False)
+
     if finetuning_args.finetuning_type == "freeze" and is_trainable:
         logger.info("Fine-tuning method: Freeze")
         num_layers = (
@@ -180,6 +184,9 @@ def init_adapter(
             if finetuning_args.use_llama_pro:
                 target_modules = find_expanded_modules(model, target_modules, finetuning_args.num_layer_trainable)
 
+            if model_args.visual_inputs:
+                target_modules = filter_vision_tower_linear(target_modules)
+
             if (
                 finetuning_args.use_dora
                 and getattr(model, "quantization_method", None) is not None

src/llamafactory/model/loader.py

@@ -163,11 +163,6 @@ def load_model(
     else:
         model.train()
 
-    if model_args.visual_inputs and model_args.tune_mm_proj:
-        lm_params = [param for name, param in model.named_parameters() if "language_model" in name]
-        for param in lm_params:
-            param.requires_grad_(False)
-
     trainable_params, all_param = count_parameters(model)
     if is_trainable:
         param_stats = "trainable params: {:d} || all params: {:d} || trainable%: {:.4f}".format(

src/llamafactory/model/utils/visual.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Tuple
+from typing import TYPE_CHECKING, Tuple, List
 
 import torch
 import transformers.models
@@ -82,3 +82,8 @@ def configure_visual_model(config: "PretrainedConfig") -> None:
     if getattr(config, "is_yi_vl_derived_model", None):
         logger.info("Detected Yi-VL model, applying projector patch.")
         transformers.models.llava.modeling_llava.LlavaMultiModalProjector = LlavaMultiModalProjectorForYiVL
+
+
+def filter_vision_tower_linear(target_modules: List[str]) -> str:
+    target_modules = f"^(?!.*vision_tower).*(?:{'|'.join(target_modules)}).*"
+    return target_modules