support llava-next(video)

Former-commit-id: 27e94593ac467e56e3a7f5c64f4ff6cee81f4b47

src/llamafactory/model/loader.py

@@ -107,7 +107,8 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule":
             setattr(processor, "video_factor", 2)
         else:
             setattr(processor, "video_factor", 1)
-    except Exception:
+    except Exception as e:
+        print(e)
         processor = None
 
     # Avoid load tokenizer, see:
@@ -123,6 +124,12 @@ def load_config(model_args: "ModelArguments") -> "PretrainedConfig":
     Loads model config.
     """
     init_kwargs = _get_init_kwargs(model_args)
+    if "LLaVA-NeXT-Video" in model_args.model_name_or_path:
+        from transformers import PretrainedConfig, LlavaNextVideoConfig, CLIPVisionConfig, LlamaConfig
+        official_config = PretrainedConfig.from_pretrained(model_args.model_name_or_path, **init_kwargs)
+        config = LlavaNextVideoConfig(CLIPVisionConfig(**official_config.vision_config), LlamaConfig(**official_config.text_config))
+        setattr(config, "visual_inputs", True)
+        return config
     return AutoConfig.from_pretrained(model_args.model_name_or_path, **init_kwargs)
 
 
@@ -159,6 +166,9 @@ def load_model(
                 load_class = AutoModelForVision2Seq
             else:
                 load_class = AutoModelForCausalLM
+                if "llava_next_video" == getattr(config, "model_type"):
+                    from transformers import LlavaNextVideoForConditionalGeneration
+                    load_class = LlavaNextVideoForConditionalGeneration
 
             if model_args.train_from_scratch:
                 model = load_class.from_config(config)

src/llamafactory/model/model_utils/misc.py

@@ -34,7 +34,7 @@ def find_all_linear_modules(model: "PreTrainedModel", freeze_vision_tower: bool)
         forbidden_modules.add("output_layer")
     elif model_type == "internlm2":
         forbidden_modules.add("output")
-    elif model_type in ["llava", "paligemma"]:
+    elif model_type in ["idefics2", "llava", "llava_next", "llava_next_video", "paligemma", "video_llava"]:
         forbidden_modules.add("multi_modal_projector")
     elif model_type == "qwen2_vl":
         forbidden_modules.add("merger")

src/llamafactory/model/model_utils/visual.py

@@ -108,7 +108,7 @@ def configure_visual_model(config: "PretrainedConfig") -> None:
     Patches VLMs before loading them.
     """
     model_type = getattr(config, "model_type", None)
-    if model_type == "llava":  # required for ds zero3 and valuehead models
+    if model_type in ["llava", "llava_next", "video_llava", "idefics2", "llava_next_video"]:  # required for ds zero3 and valuehead models
         setattr(config, "hidden_size", getattr(config.text_config, "hidden_size", None))
 
     if getattr(config, "is_yi_vl_derived_model", None):
@@ -150,7 +150,7 @@ def get_image_seqlen(config: "PretrainedConfig") -> int:
             image_seqlen += 1
     elif model_type == "paligemma":
         image_seqlen = config.vision_config.num_image_tokens
-    elif model_type == "qwen2_vl":  # variable length
+    else:
         image_seqlen = -1
 
     return image_seqlen