auto set chat template
Former-commit-id: d8bf2f0efe6919990c7032aaa06010980cfde019
This commit is contained in:
@@ -156,6 +156,9 @@ def get_dataset(
|
||||
dataset = dataset.to_iterable_dataset()
|
||||
return dataset
|
||||
|
||||
if data_args.streaming:
|
||||
raise ValueError("Turn off `streaming` when saving dataset to disk.")
|
||||
|
||||
with training_args.main_process_first(desc="load dataset"):
|
||||
all_datasets = []
|
||||
for dataset_attr in get_dataset_list(data_args):
|
||||
|
||||
Reference in New Issue
Block a user