ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[trainer] update config (#7174)

Browse Source 
Former-commit-id: 9f535d0e3c4ee3cd0f1b65218c2eee5d03f43c6f
This commit is contained in:
hoshi-hiyouga
2025-03-05 23:32:54 +08:00
committed by GitHub
parent fd0357a26d
commit 7b985f55db
13 changed files with 49 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
examples/train_lora/llama3_lora_dpo.yaml
View File

@@ -18,6 +18,7 @@ cutoff_len: 2048
max_samples: 1000
overwrite_cache: true
preprocessing_num_workers: 16
dataloader_num_workers: 4
### output
output_dir: saves/llama3-8b/lora/dpo
@@ -25,6 +26,7 @@ logging_steps: 10
save_steps: 500
plot_loss: true
overwrite_output_dir: true
save_only_model: false
### train
per_device_train_batch_size: 1
@@ -35,6 +37,7 @@ lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null
### eval
# eval_dataset: dpo_en_demo

3
examples/train_lora/llama3_lora_pretrain.yaml
View File

@@ -15,6 +15,7 @@ cutoff_len: 2048
max_samples: 1000
overwrite_cache: true
preprocessing_num_workers: 16
dataloader_num_workers: 4
### output
output_dir: saves/llama3-8b/lora/pretrain
@@ -22,6 +23,7 @@ logging_steps: 10
save_steps: 500
plot_loss: true
overwrite_output_dir: true
save_only_model: false
### train
per_device_train_batch_size: 1
@@ -32,6 +34,7 @@ lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null
### eval
# eval_dataset: c4_demo

3
examples/train_lora/llama3_lora_reward.yaml
View File

@@ -16,6 +16,7 @@ cutoff_len: 2048
max_samples: 1000
overwrite_cache: true
preprocessing_num_workers: 16
dataloader_num_workers: 4
### output
output_dir: saves/llama3-8b/lora/reward
@@ -23,6 +24,7 @@ logging_steps: 10
save_steps: 500
plot_loss: true
overwrite_output_dir: true
save_only_model: false
### train
per_device_train_batch_size: 1
@@ -33,6 +35,7 @@ lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null
### eval
# eval_dataset: dpo_en_demo

3
examples/train_lora/llama3_lora_sft.yaml
View File

@@ -16,6 +16,7 @@ cutoff_len: 2048
max_samples: 1000
overwrite_cache: true
preprocessing_num_workers: 16
dataloader_num_workers: 4
### output
output_dir: saves/llama3-8b/lora/sft
@@ -23,6 +24,7 @@ logging_steps: 10
save_steps: 500
plot_loss: true
overwrite_output_dir: true
save_only_model: false
### train
per_device_train_batch_size: 1
@@ -33,6 +35,7 @@ lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null
### eval
# eval_dataset: alpaca_en_demo

4
examples/train_lora/llama3_lora_sft_ds3.yaml
View File

@@ -17,6 +17,7 @@ cutoff_len: 2048
max_samples: 1000
overwrite_cache: true
preprocessing_num_workers: 16
dataloader_num_workers: 4
### output
output_dir: saves/llama3-8b/lora/sft
@@ -24,6 +25,7 @@ logging_steps: 10
save_steps: 500
plot_loss: true
overwrite_output_dir: true
save_only_model: false
### train
per_device_train_batch_size: 1
@@ -34,8 +36,10 @@ lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null
### eval
# eval_dataset: alpaca_en_demo
# val_size: 0.1
# per_device_eval_batch_size: 1
# eval_strategy: steps

5
examples/train_lora/llama3_lora_sft_ray.yaml
View File

@@ -17,6 +17,7 @@ cutoff_len: 2048
max_samples: 1000
overwrite_cache: true
preprocessing_num_workers: 16
dataloader_num_workers: 4
### output
output_dir: tmp_dir
@@ -24,9 +25,11 @@ logging_steps: 10
save_steps: 500
plot_loss: true
overwrite_output_dir: true
save_only_model: false
### ray
ray_run_name: llama3_8b_sft_lora
ray_storage_path: ./saves
ray_num_workers: 4 # number of GPUs to use
resources_per_worker:
GPU: 1
@@ -41,8 +44,10 @@ lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null
### eval
# eval_dataset: alpaca_en_demo
# val_size: 0.1
# per_device_eval_batch_size: 1
# eval_strategy: steps

3
examples/train_lora/llava1_5_lora_sft.yaml
View File

@@ -16,6 +16,7 @@ cutoff_len: 2048
max_samples: 1000
overwrite_cache: true
preprocessing_num_workers: 16
dataloader_num_workers: 4
### output
output_dir: saves/llava1_5-7b/lora/sft
@@ -23,6 +24,7 @@ logging_steps: 10
save_steps: 500
plot_loss: true
overwrite_output_dir: true
save_only_model: false
### train
per_device_train_batch_size: 1
@@ -33,6 +35,7 @@ lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null
### eval
# val_size: 0.1

3
examples/train_lora/qwen2vl_lora_dpo.yaml
View File

@@ -20,6 +20,7 @@ cutoff_len: 2048
max_samples: 1000
overwrite_cache: true
preprocessing_num_workers: 16
dataloader_num_workers: 4
### output
output_dir: saves/qwen2_vl-7b/lora/dpo
@@ -27,6 +28,7 @@ logging_steps: 10
save_steps: 500
plot_loss: true
overwrite_output_dir: true
save_only_model: false
### train
per_device_train_batch_size: 1
@@ -37,6 +39,7 @@ lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null
### eval
# val_size: 0.1

3
examples/train_lora/qwen2vl_lora_sft.yaml
View File

@@ -18,6 +18,7 @@ cutoff_len: 2048
max_samples: 1000
overwrite_cache: true
preprocessing_num_workers: 16
dataloader_num_workers: 4
### output
output_dir: saves/qwen2_vl-7b/lora/sft
@@ -25,6 +26,7 @@ logging_steps: 10
save_steps: 500
plot_loss: true
overwrite_output_dir: true
save_only_model: false
### train
per_device_train_batch_size: 1
@@ -35,6 +37,7 @@ lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null
### eval
# val_size: 0.1