ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix some errors due to inconsistency of model cards

Browse Source 
Former-commit-id: dd83265b9b8768eb8732f59ace128dfe4aac1c47
This commit is contained in:
Kingsley
2024-09-30 19:58:34 +08:00
parent 32b5364051
commit 8fd84c375e
5 changed files with 66 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/llamafactory/model/model_utils/visual.py
View File

@@ -92,12 +92,10 @@ def autocast_projector_dtype(model: "PreTrainedModel", model_args: "ModelArgumen
if getattr(model, "quantization_method", None):
model_type = getattr(model.config, "model_type", None)
if model_type in ["llava", "llava_next", "llava_next_video", "paligemma", "video_llava"]:
if model_type in ["llava", "llava_next", "llava_next_video", "paligemma", "pixtral", "video_llava"]:
mm_projector: "torch.nn.Module" = getattr(model, "multi_modal_projector")
elif model_type == "qwen2_vl":
mm_projector: "torch.nn.Module" = getattr(getattr(model, "visual"), "merger")
elif model_type == "pixtral":
mm_projector: "torch.nn.Module" = getattr(model, "vision_language_adapte")
else:
return
@@ -133,7 +131,6 @@ def get_forbidden_modules(config: "PretrainedConfig", finetuning_args: "Finetuni
if model_type in ["llava", "llava_next", "llava_next_video", "paligemma", "pixtral", "video_llava"]:
if finetuning_args.freeze_vision_tower:
forbidden_modules.add("vision_tower")
forbidden_modules.add("vision_encoder")
if finetuning_args.train_mm_proj_only:
forbidden_modules.add("language_model")