support streaming data, fix #284 #274 #268

Former-commit-id: 819cc1353599e5fa45658bc56dd0dbe4b258b197

src/llmtuner/tuner/sft/metric.py

@@ -1,7 +1,6 @@
 import numpy as np
 from dataclasses import dataclass
-from typing import Dict, Sequence, Tuple, Union
-from transformers.tokenization_utils import PreTrainedTokenizer
+from typing import TYPE_CHECKING, Dict, Sequence, Tuple, Union
 
 import jieba
 from rouge_chinese import Rouge
@@ -9,6 +8,9 @@ from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
 
 from llmtuner.extras.constants import IGNORE_INDEX
 
+if TYPE_CHECKING:
+    from transformers.tokenization_utils import PreTrainedTokenizer
+
 
 @dataclass
 class ComputeMetrics:
@@ -16,7 +18,7 @@ class ComputeMetrics:
     Wraps the tokenizer into metric functions, used in Seq2SeqPeftTrainer.
     """
 
-    tokenizer: PreTrainedTokenizer
+    tokenizer: "PreTrainedTokenizer"
 
     def __call__(self, eval_preds: Sequence[Union[np.ndarray, Tuple[np.ndarray]]]) -> Dict[str, float]:
         r"""