update evaluator

Former-commit-id: bb8661e62481ff7027b8969f3d8a6a17290c9da3
This commit is contained in:
hiyouga
2024-06-10 23:56:00 +08:00
parent 784088db3f
commit 4b2b92fd9a
3 changed files with 81 additions and 9 deletions

View File

@@ -10,7 +10,6 @@ class EvalTemplate:
system: str
choice: str
answer: str
prefix: str
def _parse_example(self, example: Dict[str, str]) -> Tuple[str, str]:
r"""
@@ -42,8 +41,8 @@ class EvalTemplate:
eval_templates: Dict[str, "EvalTemplate"] = {}
def _register_eval_template(name: str, system: str, choice: str, answer: str, prefix: str) -> None:
eval_templates[name] = EvalTemplate(system=system, choice=choice, answer=answer, prefix=prefix)
def _register_eval_template(name: str, system: str, choice: str, answer: str) -> None:
eval_templates[name] = EvalTemplate(system=system, choice=choice, answer=answer)
def get_eval_template(name: str) -> "EvalTemplate":
@@ -56,8 +55,7 @@ _register_eval_template(
name="en",
system="The following are multiple choice questions (with answers) about {subject}.\n\n",
choice="\n{choice}. {content}",
answer="\nAnswer: ",
prefix=" ",
answer="\nAnswer:",
)
@@ -66,5 +64,4 @@ _register_eval_template(
system="以下是中国关于{subject}考试的单项选择题,请选出其中的正确答案。\n\n",
choice="\n{choice}. {content}",
answer="\n答案:",
prefix=" ",
)