remove visual_inputs, fix qlora

Former-commit-id: be30c01c4f1482520ece770bd54c6a4837c26f0a

src/llamafactory/model/model_utils/visual.py

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Tuple
+from typing import TYPE_CHECKING, List, Sequence, Set, Tuple, Union
 
 import torch
 import transformers.models
@@ -28,7 +28,7 @@ from ...extras.logging import get_logger
 if TYPE_CHECKING:
     from transformers import LlavaConfig, PretrainedConfig, PreTrainedModel
 
-    from ...hparams import ModelArguments
+    from ...hparams import FinetuningArguments, ModelArguments
 
 
 logger = get_logger(__name__)
@@ -80,24 +80,74 @@ class LlavaMultiModalProjectorForYiVLForVLLM(LlavaMultiModalProjectorForYiVL):
         self.act = ACT2FN[projector_hidden_act]
 
 
-def autocast_projector_dtype(
-    model: "PreTrainedModel", model_args: "ModelArguments", mm_projector_name: str = "multi_modal_projector"
-) -> None:
+def autocast_projector_dtype(model: "PreTrainedModel", model_args: "ModelArguments") -> None:
+    r"""
+    Casts projector output to half precision for quantized VLMs.
+    """
+
     def _mm_projector_forward_post_hook(
         module: "torch.nn.Module", args: Tuple["torch.Tensor"], output: "torch.Tensor"
     ) -> "torch.Tensor":
         return output.to(model_args.compute_dtype)
 
-    if hasattr(model, mm_projector_name) and getattr(model, "quantization_method", None):
+    if getattr(model, "quantization_method", None):
+        if getattr(model.config, "model_type", None) in ["llava", "paligemma"]:
+            mm_projector: "torch.nn.Module" = getattr(model, "multi_modal_projector")
+        elif getattr(model.config, "model_type", None) == "qwen2_vl":
+            mm_projector: "torch.nn.Module" = getattr(getattr(model, "visual"), "merger")
+        else:
+            return
+
         logger.info("Casting multimodal projector outputs in {}.".format(model_args.compute_dtype))
-        mm_projector: "torch.nn.Module" = getattr(model, mm_projector_name)
         mm_projector.register_forward_hook(_mm_projector_forward_post_hook)
 
 
 def configure_visual_model(config: "PretrainedConfig") -> None:
+    r"""
+    Patches VLMs before loading them.
+    """
     if getattr(config, "model_type", None) == "llava":  # required for ds zero3 and valuehead models
         setattr(config, "hidden_size", getattr(config.text_config, "hidden_size", None))
 
     if getattr(config, "is_yi_vl_derived_model", None):
         logger.info("Detected Yi-VL model, applying projector patch.")
         transformers.models.llava.modeling_llava.LlavaMultiModalProjector = LlavaMultiModalProjectorForYiVL
+
+
+def get_forbidden_modules(config: "PretrainedConfig", finetuning_args: "FinetuningArguments") -> Set[str]:
+    r"""
+    Freezes vision tower and language model for VLM full/freeze tuning.
+    """
+    forbidden_modules = set()
+    if getattr(config, "model_type", None) in ["llava", "paligemma"]:
+        if finetuning_args.freeze_vision_tower:
+            forbidden_modules.add("vision_tower")
+
+        if finetuning_args.train_mm_proj_only:
+            forbidden_modules.add("language_model")
+
+    elif getattr(config, "model_type", None) == "qwen2_vl":
+        if finetuning_args.freeze_vision_tower:
+            forbidden_modules.add("visual")
+
+        if finetuning_args.train_mm_proj_only:
+            raise ValueError("Qwen2-VL models do not support `train_mm_proj_only`.")
+
+    return forbidden_modules
+
+
+def patch_target_modules(
+    config: "PretrainedConfig", finetuning_args: "FinetuningArguments", target_modules: Sequence[str]
+) -> Union[str, List[str]]:
+    r"""
+    Freezes vision tower for VLM LoRA tuning.
+    """
+    if not finetuning_args.freeze_vision_tower:
+        return target_modules
+
+    if getattr(config, "model_type", None) in ["llava", "paligemma"]:
+        return "^(?!.*vision_tower).*(?:{}).*".format("|".join(target_modules))
+    elif getattr(config, "model_type", None) == "qwen2_vl":
+        return "^(?!.*visual).*(?:{}).*".format("|".join(target_modules))
+    else:
+        return target_modules