remove checksum and fix ui args

Former-commit-id: 0cfdeb1d30efb63211434bc4656bceb59e666289
This commit is contained in:
hiyouga
2024-05-12 01:10:30 +08:00
parent 1ce400bddf
commit 8f1ba07b30
7 changed files with 11 additions and 27 deletions

View File

@@ -11,7 +11,7 @@ from .aligner import align_dataset
from .parser import get_dataset_list
from .preprocess import get_preprocess_and_print_func
from .template import get_template_and_fix_tokenizer
from .utils import checksum, merge_dataset
from .utils import merge_dataset
if TYPE_CHECKING:
@@ -61,8 +61,6 @@ def load_single_dataset(
if data_path is None:
raise ValueError("File extension must be txt, csv, json or jsonl.")
checksum(data_files, dataset_attr.file_sha1)
else:
raise NotImplementedError