ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update readme

Browse Source 
Former-commit-id: 41728fd74de7bec0cc6135aef9dfa3ae9fe7af73
This commit is contained in:
hiyouga
2024-04-26 05:44:30 +08:00
parent 23b881bff1
commit 7773ac0ead
5 changed files with 16 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/length_cdf.py
View File

@@ -32,8 +32,8 @@ def length_cdf(
overwrite_cache=True,
)
)
tokenizer = load_tokenizer(model_args)
trainset = get_dataset(tokenizer, model_args, data_args, training_args, stage="sft")
tokenizer_module = load_tokenizer(model_args)
trainset = get_dataset(model_args, data_args, training_args, stage="sft", **tokenizer_module)
total_num = len(trainset)
length_dict = defaultdict(int)
for sample in tqdm(trainset["input_ids"]):