update webui

Former-commit-id: 343a4cd82b07a40f96ba413d1d991419ff07a24a
2023-08-09 00:26:11 +08:00
parent 6e27a9e39a
commit d2e1fe9b1d
5 changed files with 24 additions and 6 deletions
--- a/src/llmtuner/extras/template.py
+++ b/src/llmtuner/extras/template.py
@@ -185,6 +185,7 @@ def get_template_and_fix_tokenizer(
    if tokenizer.pad_token_id is None and tokenizer.eos_token_id is not None:
        tokenizer.pad_token = tokenizer.eos_token

+    tokenizer.add_special_tokens(dict(additional_special_tokens=template.stop_words))
    return template