improve KTO impl., replace datasets

Former-commit-id: e56a57ddcf061de6e4acc8679f7dbf0b68364986
2024-05-18 03:44:56 +08:00
parent e4570e28a8
commit 2bff90719b
53 changed files with 448 additions and 330 deletions
--- a/data/README.md
+++ b/data/README.md
@@ -19,7 +19,10 @@ If you are using a custom dataset, please add your **dataset description** to `d
    "messages": "the column name in the dataset containing the messages. (default: conversations)",
    "system": "the column name in the dataset containing the system prompts. (default: None)",
    "tools": "the column name in the dataset containing the tool description. (default: None)",
-    "images": "the column name in the dataset containing the image inputs. (default: None)"
+    "images": "the column name in the dataset containing the image inputs. (default: None)",
+    "chosen": "the column name in the dataset containing the chosen answers. (default: None)",
+    "rejected": "the column name in the dataset containing the rejected answers. (default: None)",
+    "kto_tag": "the column name in the dataset containing the kto tags. (default: None)"
  },
  "tags (optional, used for the sharegpt format)": {
    "role_tag": "the key in the message represents the identity. (default: from)",
@@ -42,13 +45,13 @@ Currently we support dataset in **alpaca** or **sharegpt** format, the dataset i
 ```json
 [
  {
-    "instruction": "user instruction (required)",
-    "input": "user input (optional)",
+    "instruction": "human instruction (required)",
+    "input": "human input (optional)",
    "output": "model response (required)",
    "system": "system prompt (optional)",
    "history": [
-      ["user instruction in the first round (optional)", "model response in the first round (optional)"],
-      ["user instruction in the second round (optional)", "model response in the second round (optional)"]
+      ["human instruction in the first round (optional)", "model response in the first round (optional)"],
+      ["human instruction in the second round (optional)", "model response in the second round (optional)"]
    ]
  }
 ]
@@ -69,7 +72,7 @@ Regarding the above dataset, the description in `dataset_info.json` should be:
 }
 ```

-The `query` column will be concatenated with the `prompt` column and used as the user prompt, then the user prompt would be `prompt\nquery`. The `response` column represents the model response.
+The `query` column will be concatenated with the `prompt` column and used as the human prompt, then the human prompt would be `prompt\nquery`. The `response` column represents the model response.

 The `system` column will be used as the system prompt. The `history` column is a list consisting string tuples representing prompt-response pairs in the history. Note that the responses in the history **will also be used for training** in supervised fine-tuning.

@@ -98,12 +101,10 @@ For the **preference datasets**, the `response` column should be a string list w
 ```json
 [
  {
-    "instruction": "user instruction",
-    "input": "user input",
-    "output": [
-      "chosen answer",
-      "rejected answer"
-    ]
+    "instruction": "human instruction",
+    "input": "human input",
+    "chosen": "chosen answer",
+    "rejected": "rejected answer"
  }
 ]
 ```
@@ -117,7 +118,8 @@ Regarding the above dataset, the description in `dataset_info.json` should be:
  "columns": {
    "prompt": "instruction",
    "query": "input",
-    "response": "output",
+    "chosen": "chosen",
+    "rejected": "rejected"
  }
 }
 ```
@@ -132,7 +134,7 @@ The dataset in **sharegpt** format should follow the below format:
    "conversations": [
      {
        "from": "human",
-        "value": "user instruction"
+        "value": "human instruction"
      },
      {
        "from": "gpt",
@@ -179,7 +181,7 @@ We also supports the dataset in the **openai** format:
      },
      {
        "role": "user",
-        "content": "user instruction"
+        "content": "human instruction"
      },
      {
        "role": "assistant",
--- a/data/README_zh.md
+++ b/data/README_zh.md
@@ -19,7 +19,10 @@
    "messages": "数据集代表消息列表的表头名称（默认：conversations）",
    "system": "数据集代表系统提示的表头名称（默认：None）",
    "tools": "数据集代表工具描述的表头名称（默认：None）",
-    "images": "数据集代表图像输入的表头名称（默认：None）"
+    "images": "数据集代表图像输入的表头名称（默认：None）",
+    "chosen": "数据集代表更优回复的表头名称（默认：None）",
+    "rejected": "数据集代表更差回复的表头名称（默认：None）",
+    "kto_tag": "数据集代表 KTO 标签的表头名称（默认：None）"
  },
  "tags（可选，用于 sharegpt 格式）": {
    "role_tag": "消息中代表发送者身份的键名（默认：from）",
@@ -42,8 +45,8 @@
 ```json
 [
  {
-    "instruction": "用户指令（必填）",
-    "input": "用户输入（选填）",
+    "instruction": "人类指令（必填）",
+    "input": "人类输入（选填）",
    "output": "模型回答（必填）",
    "system": "系统提示词（选填）",
    "history": [
@@ -69,7 +72,7 @@
 }
 ```

-其中 `query` 列对应的内容会与 `prompt` 列对应的内容拼接后作为用户指令，即用户指令为 `prompt\nquery`。`response` 列对应的内容为模型回答。
+其中 `query` 列对应的内容会与 `prompt` 列对应的内容拼接后作为人类指令，即人类指令为 `prompt\nquery`。`response` 列对应的内容为模型回答。

 `system` 列对应的内容将被作为系统提示词。`history` 列是由多个字符串二元组构成的列表，分别代表历史消息中每轮的指令和回答。注意在指令监督学习时，历史消息中的回答**也会被用于训练**。

@@ -98,12 +101,10 @@
 ```json
 [
  {
-    "instruction": "用户指令",
-    "input": "用户输入",
-    "output": [
-      "优质回答",
-      "劣质回答"
-    ]
+    "instruction": "人类指令",
+    "input": "人类输入",
+    "chosen": "优质回答",
+    "rejected": "劣质回答"
  }
 ]
 ```
@@ -117,7 +118,8 @@
  "columns": {
    "prompt": "instruction",
    "query": "input",
-    "response": "output",
+    "chosen": "chosen",
+    "rejected": "rejected"
  }
 }
 ```
@@ -132,7 +134,7 @@
    "conversations": [
      {
        "from": "human",
-        "value": "用户指令"
+        "value": "人类指令"
      },
      {
        "from": "gpt",
@@ -165,7 +167,7 @@
 }
 ```

-其中 `messages` 列应当是一个列表，且符合 `用户/模型/用户/模型/用户/模型` 的顺序。
+其中 `messages` 列应当是一个列表，且符合 `人类/模型/人类/模型/人类/模型` 的顺序。

 我们同样支持 **openai** 格式的数据集：

@@ -179,7 +181,7 @@
      },
      {
        "role": "user",
-        "content": "用户指令"
+        "content": "人类指令"
      },
      {
        "role": "assistant",
--- a/data/alpaca_data_en_52k.json.REMOVED.git-id
+++ b/data/alpaca_data_en_52k.json.REMOVED.git-id
@@ -1 +0,0 @@
-3779ddbc040543ab1834ef216c983d6fcc06cc9a
--- a/data/alpaca_data_zh_51k.json.REMOVED.git-id
+++ b/data/alpaca_data_zh_51k.json.REMOVED.git-id
@@ -1 +0,0 @@
-a97cf9475291591843976554878568e046d8a46d
--- a/data/alpaca_gpt4_data_en.json.REMOVED.git-id
+++ b/data/alpaca_gpt4_data_en.json.REMOVED.git-id
@@ -1 +0,0 @@
-25508714b7879a1e5a6764ba7f979a980f549f1a
--- a/data/alpaca_gpt4_data_zh.json.REMOVED.git-id
+++ b/data/alpaca_gpt4_data_zh.json.REMOVED.git-id
@@ -1 +0,0 @@
-7cb6a7d11455bddc3d495750a2392683d775b184
--- a/data/comparison_gpt4_data_en.json.REMOVED.git-id
+++ b/data/comparison_gpt4_data_en.json.REMOVED.git-id
@@ -1 +0,0 @@
-f5cb08305ff5dc9c17a09809c54c8c8834aadc70
--- a/data/comparison_gpt4_data_zh.json.REMOVED.git-id
+++ b/data/comparison_gpt4_data_zh.json.REMOVED.git-id
@@ -1 +0,0 @@
-aee47b7b443496e37808d7f34ef10403ff99bcc3
--- a/data/example_dataset/example_dataset.py
+++ b/data/example_dataset/example_dataset.py
@@ -1,37 +0,0 @@
-import json
-from typing import Any, Dict, Generator, List, Tuple
-
-import datasets
-
-
-_DESCRIPTION = "An example of dataset."
-_CITATION = ""
-_HOMEPAGE = ""
-_LICENSE = ""
-_URL = "examples.json"
-
-
-class ExampleDataset(datasets.GeneratorBasedBuilder):
-    VERSION = datasets.Version("0.0.0")
-
-    def _info(self) -> datasets.DatasetInfo:
-        features = datasets.Features(
-            {
-                "instruction": datasets.Value("string"),
-                "input": datasets.Value("string"),
-                "output": datasets.Value("string"),
-                "history": datasets.Sequence(datasets.Sequence(datasets.Value("string"))),
-            }
-        )
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION, features=features, homepage=_HOMEPAGE, license=_LICENSE, citation=_CITATION
-        )
-
-    def _split_generators(self, dl_manager: datasets.DownloadManager) -> List[datasets.SplitGenerator]:
-        file_path = dl_manager.download(_URL)
-        return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": file_path})]
-
-    def _generate_examples(self, filepath: str) -> Generator[Tuple[int, Dict[str, Any]], None, None]:
-        example_dataset = json.load(open(filepath, "r", encoding="utf-8"))
-        for key, example in enumerate(example_dataset):
-            yield key, example
--- a/data/glaive_toolcall_10k.json.REMOVED.git-id
+++ b/data/glaive_toolcall_10k.json.REMOVED.git-id
@@ -1 +0,0 @@
-4748dff00d1dc42768a5b6cc772143c313017812
--- a/data/hh_rlhf_en/hh_rlhf_en.py
+++ b/data/hh_rlhf_en/hh_rlhf_en.py
@@ -79,5 +79,5 @@ class HhRlhfEn(datasets.GeneratorBasedBuilder):
                            break
                        prompt = prompt[:human_idx]

-                    yield key, {"instruction": query, "output": [r_accept, r_reject], "history": history}
+                    yield key, {"instruction": query, "chosen": r_accept, "rejected": r_reject, "history": history}
                    key += 1
--- a/data/orca_rlhf.json.REMOVED.git-id
+++ b/data/orca_rlhf.json.REMOVED.git-id
@@ -1 +0,0 @@
-736bcedea2b24a1414765c6d69cbdafaea839f3c
--- a/data/wiki_demo.txt
+++ b/data/wiki_demo.txt
--- a/data/wiki_demo.txt.REMOVED.git-id
+++ b/data/wiki_demo.txt.REMOVED.git-id
@@ -1 +0,0 @@
-c9cf509b7fdac5490cfd6dae72c2d7b8a60af6cb