ros/LlamaFactory
1
0
Fork 0
mirror of https://github.com/hiyouga/LlamaFactory.git synced 2026-02-01 08:13:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix fsdp model loading

Browse Source 
Former-commit-id: fc6fe23cc9ae4a920a17e8268a85c1aa4ad16d3b
This commit is contained in:
hiyouga
2024-05-15 16:32:28 +08:00
parent 2c011060b1
commit d4d36e157c
2 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/llmtuner/model/utils/quantization.py
View File

@@ -7,6 +7,7 @@ import torch
from datasets import load_dataset
from transformers import BitsAndBytesConfig, GPTQConfig
from transformers.integrations import is_deepspeed_zero3_enabled
from transformers.modeling_utils import is_fsdp_enabled
from transformers.utils.versions import require_version
from ...extras.constants import FILEEXT2TYPE
@@ -133,7 +134,7 @@ def configure_quantization(
bnb_4bit_quant_storage=model_args.compute_dtype, # crucial for fsdp qlora
)
if is_deepspeed_zero3_enabled() or model_args.quantization_device_map == "auto":
if is_deepspeed_zero3_enabled() or is_fsdp_enabled() or model_args.quantization_device_map == "auto":
if model_args.quantization_bit != 4:
raise ValueError("Only 4-bit quantized model can use auto device map.")