fix tokenizer #417

Former-commit-id: 01aa678311bfd213a4b410a4e0ff09f48a0d40a1
2023-08-08 23:59:41 +08:00
parent 805478c911
commit 6e27a9e39a
4 changed files with 24 additions and 17 deletions
--- a/src/llmtuner/tuner/core/loader.py
+++ b/src/llmtuner/tuner/core/loader.py
@@ -68,8 +68,6 @@ def load_model_and_tokenizer(
        padding_side=model_args.padding_side,
        **config_kwargs
    )
-    if tokenizer.pad_token_id is None and tokenizer.eos_token_id is not None: # add pad token
-        tokenizer.pad_token = tokenizer.eos_token

    if model_args.checkpoint_dir is not None and finetuning_args.finetuning_type == "full":
        model_to_load = model_args.checkpoint_dir[0]