add template, modify datasets

Former-commit-id: 81e54beb4d0f792f4fd7f450643caaf10f2f0b7d

src/llmtuner/dsets/preprocess.py

@@ -51,9 +51,6 @@ def preprocess_dataset(
             setattr(tokenizer, "add_eos_token", True)
 
         tokenized_examples = tokenizer(examples["prompt"], **kwargs)
-        # Make sure the saved tokenizer is the same as the original
-        if hasattr(tokenizer, "add_eos_token"):  # for Baichuan2 tokenizer
-            setattr(tokenizer, "add_eos_token", add_eos_token_flag)
         concatenated_examples = {k: list(chain(*tokenized_examples[k])) for k in tokenized_examples.keys()}
         total_length = len(concatenated_examples[list(concatenated_examples.keys())[0]])
         block_size = data_args.cutoff_len
@@ -64,6 +61,9 @@ def preprocess_dataset(
             k: [t[i: i + block_size] for i in range(0, total_length, block_size)]
             for k, t in concatenated_examples.items()
         }
+        # make sure the saved tokenizer is the same as the original one
+        if hasattr(tokenizer, "add_eos_token"):
+            setattr(tokenizer, "add_eos_token", add_eos_token_flag)
         return result
 
     def preprocess_supervised_dataset(examples: Dict[str, List[Any]]) -> Dict[str, List[List[int]]]:

src/llmtuner/extras/template.py

@@ -488,7 +488,7 @@ register_template(
     ],
     system=(
         "You are a helpful, respectful and honest assistant. "
-        "Always answer as helpfully as possible, while being safe.  "
+        "Always answer as helpfully as possible, while being safe. "
         "Your answers should not include any harmful, unethical, "
         "racist, sexist, toxic, dangerous, or illegal content. "
         "Please ensure that your responses are socially unbiased and positive in nature.\n\n"
@@ -546,7 +546,7 @@ register_template(
         {"token": "<|end_of_turn|>"},
         "GPT4 Correct Assistant:"
     ],
-    system="You are a helpful assistant.",
+    system="",
     sep=[
         {"token": "<|end_of_turn|>"}
     ],
@@ -668,6 +668,43 @@ register_template(
 )
 
 
+r"""
+Supports: https://huggingface.co/wenge-research/yayi-7b
+          https://huggingface.co/wenge-research/yayi-7b-llama2
+          https://huggingface.co/wenge-research/yayi-13b-llama2
+"""
+register_template(
+    name="yayi",
+    prefix=[
+        {"token": "<|System|>"},
+        ":\n{{system}}"
+    ],
+    prompt=[
+        {"token": "<|Human|>"},
+        ":\n{{query}}\n\n",
+        {"token": "<|YaYi|>"},
+        ":"
+    ],
+    system=(
+        "You are a helpful, respectful and honest assistant named YaYi "
+        "developed by Beijing Wenge Technology Co.,Ltd. "
+        "Always answer as helpfully as possible, while being safe.  "
+        "Your answers should not include any harmful, unethical, "
+        "racist, sexist, toxic, dangerous, or illegal content. "
+        "Please ensure that your responses are socially unbiased and positive in nature.\n\n"
+        "If a question does not make any sense, or is not factually coherent, "
+        "explain why instead of answering something not correct. "
+        "If you don't know the answer to a question, please don't share false information."
+    ),
+    sep=[
+        "\n\n"
+    ],
+    stop_words=[
+        "<|End|>"
+    ]
+)
+
+
 r"""
 Supports: https://huggingface.co/HuggingFaceH4/zephyr-7b-alpha
           https://huggingface.co/HuggingFaceH4/zephyr-7b-beta