ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Slim Orca data parsing

Browse Source 
Former-commit-id: 4dca3907964d27abc2b21eb55c75676901c98912
This commit is contained in:
Mark Mueller
2024-02-08 17:52:36 +01:00
parent 96f8ccf3d5
commit 04515f6b55
2 changed files with 21 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/llmtuner/data/aligner.py
View File

@@ -53,32 +53,34 @@ def convert_sharegpt(examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr"
if len(messages) == 0:
continue
n_sys = 0
prompt = []
response = []
n_sys = 0
for turn_idx, message in enumerate(messages):
accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag, dataset_attr.assistant_tag, dataset_attr.function_tag]
if message[dataset_attr.role_tag] == "system":
if dataset_attr.system_tag and message[dataset_attr.role_tag] == dataset_attr.system_tag:
outputs["system"].append(message[dataset_attr.content_tag])
n_sys += 1
elif message[dataset_attr.role_tag] not in accept_tags:
print("sytem attr", dataset_attr.system)
print("accepted tags", accept_tags)
raise ValueError("Invalid role tag in {}.".format(messages))
n_sys = 1
if (turn_idx - n_sys) % 2 == 0:
accept_tags = [dataset_attr.user_tag, dataset_attr.observation_tag]
else:
prompt.append(
{"role": tag_mapping[message[dataset_attr.role_tag]], "content": message[dataset_attr.content_tag]}
)
accept_tags = [dataset_attr.assistant_tag, dataset_attr.function_tag]
if message[dataset_attr.role_tag] not in accept_tags:
raise ValueError("Invalid role tag in {}.".format(messages))
prompt.append(
{"role": tag_mapping[message[dataset_attr.role_tag]], "content": message[dataset_attr.content_tag]}
)
last_message = prompt.pop(-1)
response.append(last_message)
outputs["prompt"].append(prompt)
outputs["response"].append(response)
if n_sys == 0:
if not dataset_attr.system_tag:
outputs["system"].append(examples[dataset_attr.system][i] if dataset_attr.system else "")
outputs["tools"].append(examples[dataset_attr.tools][i] if dataset_attr.tools else "")
assert n_sys <= 1
return outputs