ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix chatml template #408

Browse Source 
Former-commit-id: 21e0cc3f44c35ae689b00b274391492f413725ac
This commit is contained in:
hiyouga
2023-08-08 17:44:39 +08:00
parent 15acd17716
commit cda698a67f
3 changed files with 32 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/tuner/core/loader.py
View File

@@ -68,9 +68,7 @@ def load_model_and_tokenizer(
padding_side=model_args.padding_side,
**config_kwargs
)
if tokenizer.eos_token_id is None: # fix qwen tokenizer
tokenizer.eos_token = "<|endoftext|>"
if tokenizer.pad_token_id is None: # add pad token
if tokenizer.pad_token_id is None and tokenizer.eos_token_id is not None: # add pad token
tokenizer.pad_token = tokenizer.eos_token
if model_args.checkpoint_dir is not None and finetuning_args.finetuning_type == "full":