ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix test

Browse Source 
Former-commit-id: 4899309b7cac00573215f6530bfc97d7d87d70b2
This commit is contained in:
hiyouga
2024-07-14 15:44:30 +08:00
parent b92214f78b
commit 54ba30c47f
2 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/data/test_template.py
View File

@@ -44,7 +44,7 @@ def _check_tokenization(
def _check_single_template(model_id: str, template_name: str, prompt_str: str, answer_str: str, use_fast: bool):
tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=use_fast)
tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=use_fast, token=os.environ.get("HF_TOKEN", None))
content_str = tokenizer.apply_chat_template(MESSAGES, tokenize=False).rstrip("\n") # avoid extra newline
content_ids = tokenizer.encode(content_str, add_special_tokens=False)
template = get_template_and_fix_tokenizer(tokenizer, name=template_name)