auto set chat template

Former-commit-id: d8bf2f0efe6919990c7032aaa06010980cfde019
This commit is contained in:
hiyouga
2024-03-05 02:41:20 +08:00
parent 3555b87363
commit 0229fffde5
2 changed files with 81 additions and 9 deletions

View File

@@ -156,6 +156,9 @@ def get_dataset(
dataset = dataset.to_iterable_dataset()
return dataset
if data_args.streaming:
raise ValueError("Turn off `streaming` when saving dataset to disk.")
with training_args.main_process_first(desc="load dataset"):
all_datasets = []
for dataset_attr in get_dataset_list(data_args):