fix packing for eager/sdpa attn

Former-commit-id: 735a033ceb7f2da6da71d138ea091d8a665411a9
2024-07-04 01:52:43 +08:00
parent a90c6306f8
commit 3d219b91b9
9 changed files with 51 additions and 20 deletions
--- a/src/llamafactory/extras/constants.py
+++ b/src/llamafactory/extras/constants.py
@@ -79,6 +79,7 @@ TRAINING_STAGES = {
 STAGES_USE_PAIR_DATA = {"rm", "dpo"}

 SUPPORTED_CLASS_FOR_BLOCK_DIAG_ATTN = {
+    "cohere",
    "falcon",
    "gemma",
    "gemma2",