fix #2802

Former-commit-id: 1370db270d7ba1a20468abdb29193ce7534d1b4f
2024-03-12 17:08:34 +08:00
parent 7814db1b42
commit 6c1b4aec75
2 changed files with 7 additions and 0 deletions
--- a/src/llmtuner/model/patcher.py
+++ b/src/llmtuner/model/patcher.py
@@ -291,6 +291,9 @@ def patch_config(
        if "device_map" not in init_kwargs:  # quant models cannot use auto device map
            init_kwargs["device_map"] = model_args.device_map or {"": get_current_device()}

+        if init_kwargs["device_map"] == "auto":
+            init_kwargs["offload_folder"] = model_args.offload_folder
+

 def patch_model(
    model: "PreTrainedModel", tokenizer: "PreTrainedTokenizer", model_args: "ModelArguments", is_trainable: bool