support SimPO #3900

Former-commit-id: 6b954ce60155cf8334150b795cfc4bb63ca74c8b

examples/README.md

@@ -47,7 +47,7 @@ CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/lora_single_gpu/llama3_lo
 CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/lora_single_gpu/llama3_lora_ppo.yaml
 ```
 
-#### DPO Training
+#### DPO/ORPO/SimPO Training
 
 ```bash
 CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/lora_single_gpu/llama3_lora_dpo.yaml
@@ -59,12 +59,6 @@ CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/lora_single_gpu/llama3_lo
 CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/lora_single_gpu/llama3_lora_kto.yaml
 ```
 
-#### ORPO Training
-
-```bash
-CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/lora_single_gpu/llama3_lora_orpo.yaml
-```
-
 #### Preprocess Dataset
 
 It is useful for large dataset, use `tokenized_path` in config to load the preprocessed dataset.

examples/README_zh.md

@@ -47,7 +47,7 @@ CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/lora_single_gpu/llama3_lo
 CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/lora_single_gpu/llama3_lora_ppo.yaml
 ```
 
-#### DPO 训练
+#### DPO/ORPO/SimPO 训练
 
 ```bash
 CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/lora_single_gpu/llama3_lora_dpo.yaml
@@ -59,12 +59,6 @@ CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/lora_single_gpu/llama3_lo
 CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/lora_single_gpu/llama3_lora_kto.yaml
 ```
 
-#### ORPO 训练
-
-```bash
-CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/lora_single_gpu/llama3_lora_orpo.yaml
-```
-
 #### 预处理数据集
 
 对于大数据集有帮助，在配置中使用 `tokenized_path` 以加载预处理后的数据集。

examples/lora_single_gpu/llama3_lora_dpo.yaml

@@ -6,7 +6,7 @@ stage: dpo
 do_train: true
 finetuning_type: lora
 lora_target: q_proj,v_proj
-dpo_ftx: 1.0
+pref_loss: sigmoid  # [sigmoid (dpo), orpo, simpo]
 
 ### dataset
 dataset: dpo_en_demo

examples/lora_single_gpu/llama3_lora_kto.yaml

@@ -6,7 +6,6 @@ stage: kto
 do_train: true
 finetuning_type: lora
 lora_target: q_proj,v_proj
-kto_ftx: 0.1
 
 ### dataset
 dataset: kto_en_demo

examples/lora_single_gpu/llama3_lora_orpo.yaml

@@ -1,38 +0,0 @@
-### model
-model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct
-
-### method
-stage: orpo
-do_train: true
-finetuning_type: lora
-lora_target: q_proj,v_proj
-
-### dataset
-dataset: dpo_en_demo
-template: llama3
-cutoff_len: 1024
-max_samples: 1000
-overwrite_cache: true
-preprocessing_num_workers: 16
-
-### output
-output_dir: saves/llama3-8b/lora/orpo
-logging_steps: 10
-save_steps: 500
-plot_loss: true
-overwrite_output_dir: true
-
-### train
-per_device_train_batch_size: 1
-gradient_accumulation_steps: 8
-learning_rate: 0.000005
-num_train_epochs: 3.0
-lr_scheduler_type: cosine
-warmup_steps: 0.1
-fp16: true
-
-### eval
-val_size: 0.1
-per_device_eval_batch_size: 1
-evaluation_strategy: steps
-eval_steps: 500