support loftq

Former-commit-id: e7ac2eb7f7daae17525a278ffbe2f82c0fbd8093
2023-12-12 22:47:06 +08:00
parent d9a50bf93f
commit e39bbdd287
5 changed files with 42 additions and 19 deletions
--- a/src/llmtuner/hparams/finetuning_args.py
+++ b/src/llmtuner/hparams/finetuning_args.py
@@ -55,6 +55,10 @@ class LoraArguments:
                  Phi-1.5 choices: [\"Wqkv\", \"out_proj\", \"fc1\", \"fc2\"], \
                  Others choices: the same as LLaMA."}
    )
+    loftq_init: Optional[bool] = field(
+        default=False,
+        metadata={"help": "Use LoftQ initialization for quantized LoRA fine-tuning."}
+    )
    resume_lora_training: Optional[bool] = field(
        default=True,
        metadata={"help": "Whether to resume training from the last LoRA weights or create new weights after merging them."}