[FEATURE]: ADD LORA+ ALGORITHM

Former-commit-id: c35b3c3b1e27171f8a703f88ede1dc8a84c80a56

src/llmtuner/train/sft/workflow.py

@@ -12,7 +12,7 @@ from ...model import load_model, load_tokenizer
 from ...train.sft.metric import ComputeMetrics
 from ...train.sft.trainer import CustomSeq2SeqTrainer
 from ...train.utils import create_modelcard_and_push
-from ..utils import create_custom_optimzer
+from ..utils import create_custom_optimzer, create_lora_plus_optimizer
 
 
 if TYPE_CHECKING:
@@ -51,6 +51,8 @@ def run_sft(
 
     # Initialize our Trainer
     optimizer = create_custom_optimzer(model, dataset, training_args, finetuning_args)
+    if finetuning_args.lora_lr_ratio:
+        optimizer = create_lora_plus_optimizer(model, training_args, finetuning_args)
     trainer = CustomSeq2SeqTrainer(
         model=model,
         args=training_args,