release v0.1.0

Former-commit-id: 63c8d3a17cb18f0d8a8e37bfa147daf5bdd28ea9
2023-07-18 00:18:25 +08:00
parent c08ff734a7
commit eac7f97337
30 changed files with 1513 additions and 309 deletions
--- a/src/llmtuner/extras/constants.py
+++ b/src/llmtuner/extras/constants.py
@@ -5,3 +5,27 @@ VALUE_HEAD_FILE_NAME = "value_head.bin"
 FINETUNING_ARGS_NAME = "finetuning_args.json"

 LAYERNORM_NAMES = ["norm", "ln_f", "ln_attn", "ln_mlp"] # for LLaMA, BLOOM and Falcon settings
+
+METHODS = ["full", "freeze", "lora"]
+
+SUPPORTED_MODELS = {
+    "LLaMA-7B": "huggyllama/llama-7b",
+    "LLaMA-13B": "huggyllama/llama-13b",
+    "LLaMA-30B": "huggyllama/llama-30b",
+    "LLaMA-65B": "huggyllama/llama-65b",
+    "BLOOM-560M": "bigscience/bloom-560m",
+    "BLOOM-3B": "bigscience/bloom-3b",
+    "BLOOM-7B1": "bigscience/bloom-7b1",
+    "BLOOMZ-560M": "bigscience/bloomz-560m",
+    "BLOOMZ-3B": "bigscience/bloomz-3b",
+    "BLOOMZ-7B1-mt": "bigscience/bloomz-7b1-mt",
+    "Falcon-7B-Base": "tiiuae/falcon-7b",
+    "Falcon-7B-Chat": "tiiuae/falcon-7b-instruct",
+    "Falcon-40B-Base": "tiiuae/falcon-40b",
+    "Falcon-40B-Chat": "tiiuae/falcon-40b-instruct",
+    "Baichuan-7B": "baichuan-inc/Baichuan-7B",
+    "Baichuan-13B-Base": "baichuan-inc/Baichuan-13B-Base",
+    "Baichuan-13B-Chat": "baichuan-inc/Baichuan-13B-Chat",
+    "InternLM-7B-Base": "internlm/internlm-7b",
+    "InternLM-7B-Chat": "internlm/internlm-chat-7b"
+}
--- a/src/llmtuner/extras/logging.py
+++ b/src/llmtuner/extras/logging.py
@@ -2,6 +2,20 @@ import sys
 import logging


+class LoggerHandler(logging.Handler):
+
+    def __init__(self):
+        super().__init__()
+        self.log = ""
+
+    def emit(self, record):
+        if record.name == "httpx":
+            return
+        log_entry = self.format(record)
+        self.log += log_entry
+        self.log += "\n\n"
+
+
 def get_logger(name: str) -> logging.Logger:

    formatter = logging.Formatter(
--- a/src/llmtuner/extras/ploting.py
+++ b/src/llmtuner/extras/ploting.py
@@ -1,4 +1,5 @@
 import os
+import math
 import json
 import matplotlib.pyplot as plt
 from typing import List, Optional
@@ -10,12 +11,13 @@ from llmtuner.extras.logging import get_logger
 logger = get_logger(__name__)


-def smooth(scalars: List[float], weight: Optional[float] = 0.9) -> List[float]:
+def smooth(scalars: List[float]) -> List[float]:
    r"""
    EMA implementation according to TensorBoard.
    """
    last = scalars[0]
    smoothed = list()
+    weight = 1.8 * (1 / (1 + math.exp(-0.05 * len(scalars))) - 0.5) # a sigmoid function
    for next_val in scalars:
        smoothed_val = last * weight + (1 - weight) * next_val
        smoothed.append(smoothed_val)
--- a/src/llmtuner/extras/template.py
+++ b/src/llmtuner/extras/template.py
@@ -1,141 +1,29 @@
-from typing import List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
 from dataclasses import dataclass


+@dataclass
+class Format:
+    prefix: str
+    prompt: str
+    sep: str
+    use_history: bool
+
+
+templates: Dict[str, Format] = {}
+
+
@dataclass
 class Template:

    name: str

    def __post_init__(self):
-
-        if self.name == "vanilla":
-            r"""
-            Supports language model inference without histories.
-            """
-            self._register_template(
-                prefix="",
-                prompt="{query}",
-                sep="",
-                use_history=False
-            )
-
-        elif self.name == "default":
-            r"""
-            Default template.
-            """
-            self._register_template(
-                prefix="A chat between a curious user and an artificial intelligence assistant. "
-                       "The assistant gives helpful, detailed, and polite answers to the user's questions.",
-                prompt="Human: {query}\nAssistant: ",
-                sep="\n",
-                use_history=True
-            )
-
-        elif self.name == "alpaca":
-            r"""
-            Supports: https://huggingface.co/tatsu-lab/alpaca-7b-wdiff
-                      https://github.com/ymcui/Chinese-LLaMA-Alpaca
-            """
-            self._register_template(
-                prefix="Below is an instruction that describes a task. "
-                       "Write a response that appropriately completes the request.",
-                prompt="### Instruction:\n{query}\n\n### Response:\n",
-                sep="\n\n",
-                use_history=True
-            )
-
-        elif self.name == "vicuna":
-            r"""
-            Supports: https://huggingface.co/lmsys/vicuna-7b-delta-v1.1
-                      https://huggingface.co/lmsys/vicuna-13b-delta-v1.1
-            """
-            self._register_template(
-                prefix="A chat between a curious user and an artificial intelligence assistant. "
-                       "The assistant gives helpful, detailed, and polite answers to the user's questions.",
-                prompt="USER: {query} ASSISTANT: ",
-                sep="</s>",
-                use_history=True
-            )
-
-        elif self.name == "belle":
-            r"""
-            Supports: https://huggingface.co/BelleGroup/BELLE-LLaMA-EXT-13B
-            """
-            self._register_template(
-                prefix="",
-                prompt="Human: {query}\n\nBelle: ",
-                sep="\n\n",
-                use_history=True
-            )
-
-        elif self.name == "linly":
-            r"""
-            Supports: https://github.com/CVI-SZU/Linly
-            """
-            self._register_template(
-                prefix="",
-                prompt="User: {query}\nBot: ",
-                sep="\n",
-                use_history=True
-            )
-
-        elif self.name == "billa":
-            r"""
-            Supports: https://github.com/Neutralzz/BiLLa
-            """
-            self._register_template(
-                prefix="",
-                prompt="Human: {query}\nAssistant: ",
-                sep="\n",
-                use_history=True
-            )
-
-        elif self.name == "ziya":
-            r"""
-            Supports: https://huggingface.co/IDEA-CCNL/Ziya-LLaMA-13B-v1
-            """
-            self._register_template(
-                prefix="",
-                prompt="<human>:{query}\n<bot>:",
-                sep="\n",
-                use_history=True
-            )
-
-        elif self.name == "aquila":
-            r"""
-            Supports: https://huggingface.co/qhduan/aquilachat-7b
-            """
-            self._register_template(
-                prefix="A chat between a curious human and an artificial intelligence assistant. "
-                       "The assistant gives helpful, detailed, and polite answers to the human's questions.",
-                prompt="Human: {query}###Assistant: ",
-                sep="###",
-                use_history=True
-            )
-
-        elif self.name == "intern":
-            r"""
-            Supports: https://huggingface.co/internlm/internlm-chat-7b
-            """
-            self._register_template(
-                prefix="",
-                prompt="<|User|>:{query}<eoh>\n<|Bot|>:",
-                sep="<eoa>\n",
-                use_history=True
-            )
-
-        elif self.name == "baichuan":
-            r"""
-            Supports: https://huggingface.co/baichuan-inc/Baichuan-13B-Chat
-            """
-            self._register_template(
-                prefix="",
-                prompt="<reserved_102>{query}<reserved_103>",
-                sep="",
-                use_history=True
-            )
-
+        if self.name in templates:
+            self.prefix = templates[self.name].prefix
+            self.prompt = templates[self.name].prompt
+            self.sep = templates[self.name].sep
+            self.use_history = templates[self.name].use_history
        else:
            raise ValueError("Template {} does not exist.".format(self.name))

@@ -155,14 +43,6 @@ class Template:
        """
        return self._format_example(query, history, prefix) + [resp]

-    def _register_template(
-        self, prefix: str, prompt: str, sep: str, use_history: Optional[bool] = True
-    ) -> None:
-        self.prefix = prefix
-        self.prompt = prompt
-        self.sep = sep
-        self.use_history = use_history
-
    def _format_example(
        self, query: str, history: Optional[List[Tuple[str, str]]] = None, prefix: Optional[str] = ""
    ) -> List[str]:
@@ -179,3 +59,150 @@ class Template:
                convs.append(self.sep + self.prompt.format(query=user_query))
                convs.append(bot_resp)
        return convs[:-1] # drop last
+
+
+def register_template(name: str, prefix: str, prompt: str, sep: str, use_history: bool) -> None:
+    templates[name] = Format(
+        prefix=prefix,
+        prompt=prompt,
+        sep=sep,
+        use_history=use_history
+    )
+
+
+r"""
+Supports language model inference without histories.
+"""
+register_template(
+    name="vanilla",
+    prefix="",
+    prompt="{query}",
+    sep="",
+    use_history=False
+)
+
+
+r"""
+Default template.
+"""
+register_template(
+    name="default",
+    prefix="A chat between a curious user and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+    prompt="Human: {query}\nAssistant: ",
+    sep="\n",
+    use_history=True
+)
+
+
+r"""
+Supports: https://huggingface.co/tatsu-lab/alpaca-7b-wdiff
+          https://github.com/ymcui/Chinese-LLaMA-Alpaca
+"""
+register_template(
+    name="alpaca",
+    prefix="Below is an instruction that describes a task. "
+           "Write a response that appropriately completes the request.",
+    prompt="### Instruction:\n{query}\n\n### Response:\n",
+    sep="\n\n",
+    use_history=True
+)
+
+
+r"""
+Supports: https://huggingface.co/lmsys/vicuna-7b-delta-v1.1
+          https://huggingface.co/lmsys/vicuna-13b-delta-v1.1
+"""
+register_template(
+    name="vicuna",
+    prefix="A chat between a curious user and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+    prompt="USER: {query} ASSISTANT: ",
+    sep="</s>",
+    use_history=True
+)
+
+
+r"""
+Supports: https://huggingface.co/BelleGroup/BELLE-LLaMA-EXT-13B
+"""
+register_template(
+    name="belle",
+    prefix="",
+    prompt="Human: {query}\n\nBelle: ",
+    sep="\n\n",
+    use_history=True
+)
+
+
+r"""
+Supports: https://github.com/CVI-SZU/Linly
+"""
+register_template(
+    name="linly",
+    prefix="",
+    prompt="User: {query}\nBot: ",
+    sep="\n",
+    use_history=True
+)
+
+
+r"""
+Supports: https://github.com/Neutralzz/BiLLa
+"""
+register_template(
+    name="billa",
+    prefix="",
+    prompt="Human: {query}\nAssistant: ",
+    sep="\n",
+    use_history=True
+)
+
+
+r"""
+Supports: https://huggingface.co/IDEA-CCNL/Ziya-LLaMA-13B-v1
+"""
+register_template(
+    name="ziya",
+    prefix="",
+    prompt="<human>:{query}\n<bot>:",
+    sep="\n",
+    use_history=True
+)
+
+
+r"""
+Supports: https://huggingface.co/qhduan/aquilachat-7b
+"""
+register_template(
+    name="aquila",
+    prefix="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    prompt="Human: {query}###Assistant: ",
+    sep="###",
+    use_history=True
+)
+
+
+r"""
+Supports: https://huggingface.co/internlm/internlm-chat-7b
+"""
+register_template(
+    name="intern",
+    prefix="",
+    prompt="<|User|>:{query}<eoh>\n<|Bot|>:",
+    sep="<eoa>\n",
+    use_history=True
+)
+
+
+r"""
+Supports: https://huggingface.co/baichuan-inc/Baichuan-13B-Chat
+"""
+register_template(
+    name="baichuan",
+    prefix="",
+    prompt="<reserved_102>{query}<reserved_103>",
+    sep="",
+    use_history=True
+)