Added Mixture of Depths

Former-commit-id: 75dd98b9abc847e22cb263c17ebcd2ca5dd98345
2024-04-18 20:31:24 +02:00
parent 8397808d1d
commit 44cda2eece
10 changed files with 103 additions and 6 deletions
--- a/src/llmtuner/hparams/parser.py
+++ b/src/llmtuner/hparams/parser.py
@@ -82,6 +82,9 @@ def _check_extra_dependencies(
    if model_args.use_unsloth:
        require_version("unsloth", "Please install unsloth: https://github.com/unslothai/unsloth")

+    if model_args.mixture_of_depths:
+        require_version("mixture-of-depth", "To fix: pip install mixture-of-depth")
+
    if model_args.infer_backend == "vllm":
        require_version("vllm>=0.3.3", "To fix: pip install vllm>=0.3.3")