ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'hiyouga:main' into main

Browse Source 
Former-commit-id: 131d0bcd554dedd794add7eb3d7b1201cac80e7c
This commit is contained in:
BUAADreamer
2024-04-25 20:02:50 +08:00
committed by GitHub
parent d1d08d066a 6be321b5da
commit f74e640565
8 changed files with 58 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/data/utils.py
View File

@@ -78,9 +78,9 @@ def split_dataset(
if training_args.do_train:
if data_args.val_size > 1e-6: # Split the dataset
if data_args.streaming:
dataset = dataset.shuffle(buffer_size=data_args.buffer_size, seed=training_args.seed)
val_set = dataset.take(int(data_args.val_size))
train_set = dataset.skip(int(data_args.val_size))
dataset = dataset.shuffle(buffer_size=data_args.buffer_size, seed=training_args.seed)
return {"train_dataset": train_set, "eval_dataset": val_set}
else:
val_size = int(data_args.val_size) if data_args.val_size > 1 else data_args.val_size