refactor patcher

Former-commit-id: 263cfe1294f5c3188f5e8d65791f35ee0d87315a
2024-04-24 03:02:23 +08:00
parent d2bb1b3a6b
commit 03f2e3284a
14 changed files with 598 additions and 394 deletions
--- a/src/llmtuner/extras/constants.py
+++ b/src/llmtuner/extras/constants.py
@@ -47,6 +47,8 @@ TRAINING_STAGES = {

 STAGES_USE_PAIR_DATA = ["rm", "dpo", "orpo"]

+SUPPORTED_CLASS_FOR_S2ATTN = ["llama"]
+
 V_HEAD_WEIGHTS_NAME = "value_head.bin"

 V_HEAD_SAFE_WEIGHTS_NAME = "value_head.safetensors"