improve lora+ impl.

Former-commit-id: 332bad25455a70ad9204e7dd384bb086d789aa39
2024-03-13 23:32:51 +08:00
parent 73f4513c84
commit 46f99ff277
12 changed files with 165 additions and 169 deletions
--- a/examples/extras/lora_plus/sft.sh
+++ b/examples/extras/lora_plus/sft.sh
@@ -9,7 +9,7 @@ CUDA_VISIBLE_DEVICES=0 python ../../src/train_bash.py \
    --template default \
    --finetuning_type lora \
    --lora_target q_proj,v_proj \
-    --output_dir ../../saves/LLaMA2-7B/lora_plus/sft \
+    --output_dir ../../saves/LLaMA2-7B/loraplus/sft \
    --overwrite_cache \
    --overwrite_output_dir \
    --cutoff_len 1024 \
@@ -30,4 +30,4 @@ CUDA_VISIBLE_DEVICES=0 python ../../src/train_bash.py \
    --val_size 0.1 \
    --plot_loss \
    --fp16 \
-    --lora_lr_ratio 16.0
+    --loraplus_lr_ratio 16.0