1. add model and dataset info to support webui

Former-commit-id: 92f6226f3fecbd9af744a7232dda2c68b2bb0d86
2024-10-10 16:46:34 +08:00
parent c42dcab32b
commit aad79127e6
6 changed files with 24 additions and 5 deletions
--- a/src/llamafactory/data/loader.py
+++ b/src/llamafactory/data/loader.py
@@ -94,7 +94,7 @@ def _load_single_dataset(
                name=data_name,
                data_dir=data_dir,
                data_files=data_files,
-                split=data_args.split,
+                split=dataset_attr.split,
                cache_dir=cache_dir,
                token=model_args.ms_hub_token,
                streaming=(data_args.streaming and (dataset_attr.load_from != "file")),
--- a/src/llamafactory/data/parser.py
+++ b/src/llamafactory/data/parser.py
@@ -98,6 +98,7 @@ def get_dataset_list(dataset_names: Optional[Sequence[str]], dataset_dir: str) -
    dataset_list: List["DatasetAttr"] = []
    for name in dataset_names:
        if dataset_info is None: # dataset_dir is ONLINE
+            load_from = None
            if use_openmind():
                load_from = "om_hub"
            if use_modelscope():