improve aligner

Former-commit-id: cc7296b92e10c24967fc753393275b71d300683f
2024-02-10 16:39:19 +08:00
parent a41fa6e730
commit 1955a8ea5a
10 changed files with 80 additions and 64 deletions
--- a/src/llmtuner/data/template.py
+++ b/src/llmtuner/data/template.py
@@ -247,7 +247,7 @@ def _add_or_replace_eos_token(tokenizer: "PreTrainedTokenizer", eos_token: str)
        logger.info("Replace eos token: {}".format(tokenizer.eos_token))

    if is_oov:
-        logger.warning("New token is added, you must enable `resize_vocab` to activate it.")
+        logger.warning("New tokens have been added, make sure `resize_vocab` is True.")


 def get_template_and_fix_tokenizer(