update readme

Former-commit-id: 41728fd74de7bec0cc6135aef9dfa3ae9fe7af73
2024-04-26 05:44:30 +08:00
parent 23b881bff1
commit 7773ac0ead
5 changed files with 16 additions and 6 deletions
--- a/scripts/cal_lr.py
+++ b/scripts/cal_lr.py
@@ -44,8 +44,9 @@ def calculate_lr(
            overwrite_cache=True,
        )
    )
-    tokenizer = load_tokenizer(model_args)
-    trainset = get_dataset(tokenizer, model_args, data_args, training_args, stage)
+    tokenizer_module = load_tokenizer(model_args)
+    tokenizer = tokenizer_module["tokenizer"]
+    trainset = get_dataset(model_args, data_args, training_args, stage, **tokenizer_module)
    if stage == "pt":
        data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
    elif stage == "sft":