ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tiny fix

Browse Source 
Former-commit-id: c0e9c0484dae6db93cef5048bad827ff22b1986a
This commit is contained in:
hiyouga
2024-09-05 23:41:16 +08:00
parent c5ef52a67a
commit 27547355e6
5 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llamafactory/data/processors/pretrain.py
View File

@@ -36,7 +36,7 @@ def preprocess_pretrain_dataset(
if data_args.template == "gemma":
text_examples = [tokenizer.bos_token + example for example in text_examples]
result = tokenizer(text_examples, add_special_tokens=False, max_length=data_args.cutoff_len, truncation=True)
result = tokenizer(text_examples, add_special_tokens=False, truncation=True, max_length=data_args.cutoff_len)
else:
tokenized_examples = tokenizer(text_examples, add_special_tokens=False)
concatenated_examples = {k: list(chain(*tokenized_examples[k])) for k in tokenized_examples.keys()}