fix llava qlora

Former-commit-id: 01c5a669f6fe598aac1758a700a7607da37db1bc

src/llmtuner/model/patcher.py

@@ -15,6 +15,7 @@ from .utils.longlora import configure_longlora
 from .utils.moe import add_z3_leaf_module, configure_moe
 from .utils.quantization import configure_quantization
 from .utils.rope import configure_rope
+from .utils.visual import autocast_projector_dtype
 
 
 if TYPE_CHECKING:
@@ -92,6 +93,9 @@ def patch_model(
     if model_args.resize_vocab:
         resize_embedding_layer(model, tokenizer)
 
+    if model_args.visual_inputs:
+        autocast_projector_dtype(model, model_args)
+
     if is_trainable:
         prepare_model_for_training(model, model_args)
         add_z3_leaf_module(model)