Former-commit-id: 4e1af5a5d39d9e2f374c1372e2d67120c63fea09
This commit is contained in:
hiyouga
2023-12-09 20:53:18 +08:00
parent 2e6ed731cf
commit f3ffa8310f
4 changed files with 18 additions and 13 deletions

View File

@@ -15,6 +15,7 @@ class DatasetAttr:
dataset_sha1: Optional[str] = None
system_prompt: Optional[str] = None
subset: Optional[str] = None
folder: Optional[str] = None
ranking: Optional[bool] = False
formatting: Optional[Literal["alpaca", "sharegpt"]] = "alpaca"
@@ -173,6 +174,7 @@ class DataArguments:
dataset_attr.content = dataset_info[name]["columns"].get("content", None)
dataset_attr.subset = dataset_info[name].get("subset", None)
dataset_attr.folder = dataset_info[name].get("folder", None)
dataset_attr.ranking = dataset_info[name].get("ranking", False)
dataset_attr.formatting = dataset_info[name].get("formatting", "alpaca")
dataset_attr.system_prompt = prompt_list[i]