add MMLU and C-Eval script

Former-commit-id: 3403f876127b4b99c5e3edb2834cc3b9a3a0063f
2023-09-23 00:34:17 +08:00
parent 4fbdf18c70
commit 35d1921081
12 changed files with 579 additions and 856 deletions
--- a/tests/cal_flops.py
+++ b/tests/cal_flops.py
@@ -6,8 +6,8 @@
 import fire
 import torch
 from typing import Optional
-from deepspeed.accelerator import get_accelerator
-from deepspeed.profiling.flops_profiler import get_model_profile
+from deepspeed.accelerator import get_accelerator # type: ignore
+from deepspeed.profiling.flops_profiler import get_model_profile # type: ignore

 from llmtuner import ChatModel

@@ -35,7 +35,7 @@ def calculate(
            print_profile=True,
            detailed=True
        )
-        print("FLOPS:", flops)
+        print("FLOPs:", flops)
        print("MACs:", macs)
        print("Params:", params)