fix #2249

Former-commit-id: 7ec64588c541422875adfdaf5692a27d05b96cb9

src/llmtuner/eval/evaluator.py

@@ -4,6 +4,7 @@ import os
 import json
 import torch
 import numpy as np
+import inspect
 from tqdm import tqdm, trange
 from typing import Any, Dict, List, Optional
 
@@ -53,13 +54,18 @@ class Evaluator:
         pbar = tqdm(categorys.keys(), desc="Processing subjects", position=0)
         results = {}
         for subject in pbar:
+            if "trust_remote_code" in inspect.signature(load_dataset).parameters: # for datasets==2.16.0
+                kwargs = {"trust_remote_code": True}
+            else:
+                kwargs = {}
+
             dataset = load_dataset(
                 path=os.path.join(self.eval_args.task_dir, self.eval_args.task),
                 name=subject,
                 cache_dir=self.model_args.cache_dir,
                 download_mode=self.eval_args.download_mode,
                 token=self.model_args.hf_hub_token,
-                trust_remote_code=True
+                **kwargs
             )
             pbar.set_postfix_str(categorys[subject]["name"])
             inputs, outputs, labels = [], [], []

src/llmtuner/eval/template.py

@@ -27,8 +27,7 @@ class EvalTemplate:
         self,
         target_data: Dict[str, str],
         support_set: "Dataset",
-        subject_name: str,
-        use_history: bool
+        subject_name: str
     ) -> List[Dict[str, str]]:
         messages = []
         for k in range(len(support_set)):
@@ -39,12 +38,7 @@ class EvalTemplate:
         prompt, response = self.parse_example(target_data)
         messages.append({"role": Role.USER, "content": prompt})
         messages.append({"role": Role.ASSISTANT, "content": response})
-
         messages[0]["content"] = self.system.format(subject=subject_name) + messages[0]["content"]
-
-        if not use_history:
-            messages = [{"role": Role.USER, "content": "\n\n".join([message["content"] for message in messages[:-1]])}]
-
         return messages