ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add llava-next/llava-next-video/video-llava

Browse Source 
Former-commit-id: a4e4239931b0b0e3fd12c9f9bbfd2c201cbc78ca
This commit is contained in:
BUAADreamer
2024-09-28 00:57:03 +08:00
parent 86c84972c8
commit b6fb00e046
5 changed files with 134 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/llamafactory/model/loader.py
View File

@@ -119,15 +119,6 @@ def load_config(model_args: "ModelArguments") -> "PretrainedConfig":
Loads model config.
"""
init_kwargs = _get_init_kwargs(model_args)
if "LLaVA-NeXT-Video" in model_args.model_name_or_path:
from transformers import CLIPVisionConfig, LlamaConfig, LlavaNextVideoConfig, PretrainedConfig
official_config = PretrainedConfig.from_pretrained(model_args.model_name_or_path, **init_kwargs)
config = LlavaNextVideoConfig(
CLIPVisionConfig(**official_config.vision_config), LlamaConfig(**official_config.text_config)
)
setattr(config, "visual_inputs", True)
return config
return AutoConfig.from_pretrained(model_args.model_name_or_path, **init_kwargs)
@@ -164,11 +155,6 @@ def load_model(
load_class = AutoModelForVision2Seq
else:
load_class = AutoModelForCausalLM
if "llava_next_video" == getattr(config, "model_type"):
from transformers import LlavaNextVideoForConditionalGeneration
load_class = LlavaNextVideoForConditionalGeneration
if model_args.train_from_scratch:
model = load_class.from_config(config)
else: