[v1] support ulysses cp for fsdp2 (#10262)

This commit is contained in:
sunyi0505
2026-03-27 16:22:48 +08:00
committed by GitHub
parent df2e6edb7e
commit b5afabe3d2
8 changed files with 552 additions and 7 deletions

View File

@@ -0,0 +1,23 @@
model: Qwen/Qwen3-0.6B
trust_remote_code: true
model_class: llm
template: qwen3_nothink
# FSDP Config
dist_config:
name: fsdp2
dcp_path: null
cp_mode: ulysses
cp_size: 2
### data
train_dataset: data/v1_sft_demo.yaml
### training
output_dir: outputs/test_ulysses_cp
micro_batch_size: 1
cutoff_len: 2048
learning_rate: 1.0e-4
bf16: false
max_steps: 10