support batch_eval_metrics, fix #4826

Former-commit-id: 3fe1df17188825f8a32fbe6a1294b4b532ce0c85

src/llamafactory/hparams/finetuning_args.py

@@ -334,6 +334,10 @@ class FinetuningArguments(FreezeArguments, LoraArguments, RLHFArguments, GaloreA
         default=False,
         metadata={"help": "Whether or not to train the multimodal projector for MLLM only."},
     )
+    compute_accuracy: bool = field(
+        default=False,
+        metadata={"help": "Whether or not to compute the token-level accuracy at evaluation."},
+    )
     plot_loss: bool = field(
         default=False,
         metadata={"help": "Whether or not to save the training loss curves."},

src/llamafactory/hparams/parser.py

@@ -211,6 +211,9 @@ def get_train_args(args: Optional[Dict[str, Any]] = None) -> _TRAIN_CLS:
     if training_args.predict_with_generate and data_args.eval_dataset is None:
         raise ValueError("Cannot use `predict_with_generate` if `eval_dataset` is None.")
 
+    if training_args.predict_with_generate and finetuning_args.compute_accuracy:
+        raise ValueError("Cannot use `predict_with_generate` and `compute_accuracy` together.")
+
     if training_args.do_train and model_args.quantization_device_map == "auto":
         raise ValueError("Cannot use device map for quantized models in training.")