ros/LlamaFactory
1
0
Fork 0
mirror of https://github.com/hiyouga/LlamaFactory.git synced 2026-02-02 20:43:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

[deps] adapt to transformers v5 (#10147)

Browse Source 
Co-authored-by: frozenleaves <frozen@Mac.local>
Co-authored-by: hiyouga <hiyouga@buaa.edu.cn>
This commit is contained in:
浮梦
2026-02-02 12:07:19 +08:00
committed by GitHub
parent 762b480131
commit bf04ca6af8
23 changed files with 149 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
tests/model/model_utils/test_visual.py
View File

@@ -16,7 +16,8 @@ import os
import pytest
import torch
from transformers import AutoConfig, AutoModelForVision2Seq
from safetensors.torch import load_file
from transformers import AutoConfig, AutoModelForImageTextToText
from llamafactory.extras.packages import is_transformers_version_greater_than
from llamafactory.hparams import FinetuningArguments, ModelArguments
@@ -36,7 +37,7 @@ def test_visual_full(freeze_vision_tower: bool, freeze_multi_modal_projector: bo
)
config = AutoConfig.from_pretrained(model_args.model_name_or_path)
with torch.device("meta"):
model = AutoModelForVision2Seq.from_config(config)
model = AutoModelForImageTextToText.from_config(config)
model = init_adapter(config, model, model_args, finetuning_args, is_trainable=True)
for name, param in model.named_parameters():
@@ -56,7 +57,7 @@ def test_visual_lora(freeze_vision_tower: bool, freeze_language_model: bool):
)
config = AutoConfig.from_pretrained(model_args.model_name_or_path)
with torch.device("meta"):
model = AutoModelForVision2Seq.from_config(config)
model = AutoModelForImageTextToText.from_config(config)
model = init_adapter(config, model, model_args, finetuning_args, is_trainable=True)
trainable_params, frozen_params = set(), set()
@@ -86,13 +87,14 @@ def test_visual_model_save_load():
finetuning_args = FinetuningArguments(finetuning_type="full")
config = AutoConfig.from_pretrained(model_args.model_name_or_path)
with torch.device("meta"):
model = AutoModelForVision2Seq.from_config(config)
model = AutoModelForImageTextToText.from_config(config)
model = init_adapter(config, model, model_args, finetuning_args, is_trainable=False)
model.to_empty(device="cpu")
loaded_model_weight = dict(model.named_parameters())
model.save_pretrained(os.path.join("output", "qwen2_vl"), max_shard_size="10GB", safe_serialization=False)
saved_model_weight = torch.load(os.path.join("output", "qwen2_vl", "pytorch_model.bin"), weights_only=False)
model.save_pretrained(os.path.join("output", "qwen2_vl"), max_shard_size="10GB", safe_serialization=True)
saved_model_weight = load_file(os.path.join("output", "qwen2_vl", "model.safetensors"))
if is_transformers_version_greater_than("4.52.0"):
assert "model.language_model.layers.0.self_attn.q_proj.weight" in loaded_model_weight