alert pad_token source

Former-commit-id: f26a84e0d927d2554890daf431a93652e18f4235
2023-08-15 00:07:56 +08:00
parent 7f0b908de2
commit ef2ca0a827
2 changed files with 5 additions and 2 deletions
--- a/src/llmtuner/extras/template.py
+++ b/src/llmtuner/extras/template.py
@@ -204,7 +204,10 @@ def get_template_and_fix_tokenizer(
        logger.info("Add eos token: {}".format(tokenizer.eos_token))

    if tokenizer.pad_token_id is None:
-        tokenizer.pad_token = tokenizer.eos_token
+        if tokenizer.unk_token_id is not None:
+            tokenizer.pad_token = tokenizer.unk_token
+        else:
+            tokenizer.pad_token = tokenizer.eos_token
        logger.info("Add pad token: {}".format(tokenizer.pad_token))

    tokenizer.add_special_tokens(dict(additional_special_tokens=template.stop_words))