add vllm_dtype arg #3387 #3717

Former-commit-id: a0dd3a6351bb78541d40fec1d2fc457d803c86a4
2024-06-06 02:53:27 +08:00
parent 765715af21
commit fc95800840
2 changed files with 6 additions and 5 deletions
--- a/src/llamafactory/hparams/model_args.py
+++ b/src/llamafactory/hparams/model_args.py
@@ -125,6 +125,10 @@ class ModelArguments:
        default=8,
        metadata={"help": "Maximum rank of all LoRAs in the vLLM engine."},
    )
+    vllm_dtype: Literal["auto", "float16", "bfloat16", "float32"] = field(
+        default="auto",
+        metadata={"help": "Data type for model weights and activations in the vLLM engine."},
+    )
    offload_folder: str = field(
        default="offload",
        metadata={"help": "Path to offload model weights."},