add bf16 lora option

Former-commit-id: 58e7d7ff0cf9bf30e53b3eb12576f38d31976413
2024-01-19 16:29:03 +08:00
parent 9b390c4bea
commit 384f0e7678
2 changed files with 6 additions and 2 deletions
--- a/src/llmtuner/hparams/finetuning_args.py
+++ b/src/llmtuner/hparams/finetuning_args.py
@@ -55,9 +55,13 @@ class LoraArguments:
                  Phi choices: [\"Wqkv\", \"out_proj\", \"fc1\", \"fc2\"], \
                  Others choices: the same as LLaMA."}
    )
+    lora_bf16_mode: Optional[bool] = field(
+        default=False,
+        metadata={"help": "Whether or not to train lora adapters in bf16 precision."}
+    )
    create_new_adapter: Optional[bool] = field(
        default=False,
-        metadata={"help": "Whether to create a new adapter with randomly initialized weight or not."}
+        metadata={"help": "Whether or not to create a new adapter with randomly initialized weight."}
    )