ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve KTO impl., replace datasets

Browse Source 
Former-commit-id: e56a57ddcf061de6e4acc8679f7dbf0b68364986
This commit is contained in:
hiyouga
2024-05-18 03:44:56 +08:00
parent e4570e28a8
commit 2bff90719b
53 changed files with 448 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/extras/badam/llama3_lora_sft.yaml
View File

@@ -11,7 +11,7 @@ badam_switch_interval: 50
badam_verbose: 2
### dataset
dataset: identity,alpaca_gpt4_en
dataset: identity,alpaca_en_demo
template: llama3
cutoff_len: 1024
max_samples: 1000

2
examples/extras/fsdp_qlora/llama3_lora_sft.yaml
View File

@@ -12,7 +12,7 @@ lora_target: q_proj,v_proj
ddp_timeout: 180000000
### dataset
dataset: identity,alpaca_gpt4_en
dataset: identity,alpaca_en_demo
template: llama3
cutoff_len: 1024
max_samples: 1000

2
examples/extras/galore/llama3_full_sft.yaml
View File

@@ -12,7 +12,7 @@ galore_rank: 128
galore_scale: 2.0
### dataset
dataset: identity,alpaca_gpt4_en
dataset: identity,alpaca_en_demo
template: llama3
cutoff_len: 1024
max_samples: 1000

2
examples/extras/llama_pro/llama3_freeze_sft.yaml
View File

@@ -10,7 +10,7 @@ freeze_trainable_modules: all
use_llama_pro: true
### dataset
dataset: identity,alpaca_gpt4_en
dataset: identity,alpaca_en_demo
template: llama3
cutoff_len: 1024
max_samples: 1000

2
examples/extras/loraplus/llama3_lora_sft.yaml
View File

@@ -9,7 +9,7 @@ lora_target: q_proj,v_proj
loraplus_lr_ratio: 16.0
### dataset
dataset: identity,alpaca_gpt4_en
dataset: identity,alpaca_en_demo
template: llama3
cutoff_len: 1024
max_samples: 1000

2
examples/extras/mod/llama3_full_sft.yaml
View File

@@ -8,7 +8,7 @@ finetuning_type: full
mixture_of_depths: convert
### dataset
dataset: identity,alpaca_gpt4_en
dataset: identity,alpaca_en_demo
template: llama3
cutoff_len: 1024
max_samples: 1000