@@ -58,7 +58,11 @@ class DataCollatorWithVerbose(DataCollatorWithPadding):
|
||||
@pytest.mark.parametrize("disable_shuffling", [False, True])
|
||||
def test_shuffle(disable_shuffling: bool):
|
||||
model_args, data_args, training_args, finetuning_args, _ = get_train_args(
|
||||
{"output_dir": f"dummy_dir/{disable_shuffling}", "disable_shuffling": disable_shuffling, **TRAIN_ARGS}
|
||||
{
|
||||
"output_dir": os.path.join("output", f"shuffle{str(disable_shuffling).lower()}"),
|
||||
"disable_shuffling": disable_shuffling,
|
||||
**TRAIN_ARGS,
|
||||
}
|
||||
)
|
||||
tokenizer_module = load_tokenizer(model_args)
|
||||
tokenizer = tokenizer_module["tokenizer"]
|
||||
|
||||
Reference in New Issue
Block a user