fix mrope

Former-commit-id: 55bee1d333549ca19858b3f5c1b7b86926e5fb09
2024-12-12 15:08:17 +00:00
parent cfff136b2a
commit fb22651faf
11 changed files with 32 additions and 9 deletions
--- a/src/llamafactory/train/sft/workflow.py
+++ b/src/llamafactory/train/sft/workflow.py
@@ -56,6 +56,7 @@ def run_sft(

    data_collator = SFTDataCollatorWith4DAttentionMask(
        template=template,
+        model=model,
        pad_to_multiple_of=8 if training_args.do_train else None,  # for shift short attention
        label_pad_token_id=IGNORE_INDEX if data_args.ignore_pad_token_for_loss else tokenizer.pad_token_id,
        block_diag_attn=model_args.block_diag_attn,