remove visual_inputs, fix qlora

Former-commit-id: be30c01c4f1482520ece770bd54c6a4837c26f0a

src/llamafactory/model/loader.py

@@ -93,17 +93,10 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule":
 
     patch_tokenizer(tokenizer)
 
-    if model_args.visual_inputs:
-        try:
-            processor = AutoProcessor.from_pretrained(model_args.model_name_or_path, **init_kwargs)
-            setattr(processor, "tokenizer", tokenizer)
-        except Exception:
-            raise ValueError(
-                "This multimodal LLM is not supported.\n"
-                "Download LLaVA-1.5 models from: https://huggingface.co/llava-hf\n"
-                "Download Yi-VL models from: https://huggingface.co/BUAADreamer"
-            )
-    else:
+    try:
+        processor = AutoProcessor.from_pretrained(model_args.model_name_or_path, **init_kwargs)
+        setattr(processor, "tokenizer", tokenizer)
+    except Exception:
         processor = None
 
     return {"tokenizer": tokenizer, "processor": processor}
@@ -145,12 +138,16 @@ def load_model(
 
         if model_args.mixture_of_depths == "load":
             model = load_mod_pretrained_model(**init_kwargs)
-        elif model_args.visual_inputs:
-            model = AutoModelForVision2Seq.from_pretrained(**init_kwargs)
-        elif model_args.train_from_scratch:
-            model = AutoModelForCausalLM.from_config(config)
         else:
-            model = AutoModelForCausalLM.from_pretrained(**init_kwargs)
+            if type(config) in AutoModelForVision2Seq._model_mapping.keys():  # assume built-in models
+                load_class = AutoModelForVision2Seq
+            else:
+                load_class = AutoModelForCausalLM
+
+            if model_args.train_from_scratch:
+                model = load_class.from_config(config)
+            else:
+                model = load_class.from_pretrained(**init_kwargs)
 
         if model_args.mixture_of_depths == "convert":
             model = convert_pretrained_model_to_mod(model, config, model_args)