ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix chatml template #408

Browse Source 
Former-commit-id: 21e0cc3f44c35ae689b00b274391492f413725ac
This commit is contained in:
hiyouga
2023-08-08 17:44:39 +08:00
parent 15acd17716
commit cda698a67f
3 changed files with 32 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/llmtuner/dsets/preprocess.py
View File

@@ -31,7 +31,11 @@ def preprocess_dataset(
def preprocess_pretrain_dataset(examples: Dict[str, List[Any]]) -> Dict[str, Any]:
# build grouped texts with format `X1 X2 X3 ...` (without <eos>)
tokenized_examples = tokenizer(examples["prompt"], add_special_tokens=False)
if hasattr(tokenizer, "tokenizer"): # for tiktoken tokenizer (Qwen)
kwargs = dict(allowed_special="all")
else:
kwargs = dict(add_special_tokens=False)
tokenized_examples = tokenizer(examples["prompt"], **kwargs)
concatenated_examples = {k: list(chain(*tokenized_examples[k])) for k in tokenized_examples.keys()}
total_length = len(concatenated_examples[list(concatenated_examples.keys())[0]])
block_size = data_args.max_source_length