support galore

Former-commit-id: b67a4a46a88d83bb2a3459b3317b66cda15e0171
2024-03-07 22:41:36 +08:00
parent 5d0c95bd02
commit 1e6fb6c8aa
12 changed files with 115 additions and 16 deletions
--- a/src/llmtuner/hparams/parser.py
+++ b/src/llmtuner/hparams/parser.py
@@ -180,7 +180,7 @@ def get_train_args(args: Optional[Dict[str, Any]] = None) -> _TRAIN_CLS:

    # Post-process training arguments
    if (
-        training_args.local_rank != -1
+        training_args.parallel_mode.value == "distributed"
        and training_args.ddp_find_unused_parameters is None
        and finetuning_args.finetuning_type == "lora"
    ):