fix #1550

Former-commit-id: c12acd21a5a500892ed739c79327ccd39fddad5b

src/llmtuner/model/loader.py

@@ -168,12 +168,17 @@ def load_model_and_tokenizer(
         config_kwargs["device_map"] = {"": get_current_device()}
         logger.info("Quantizing model to {} bit.".format(model_args.quantization_bit))
 
+    if is_deepspeed_zero3_enabled() or getattr(config, "model_type", None) == "chatglm":
+        low_cpu_mem_usage = False
+    else:
+        low_cpu_mem_usage = True
+
     # Load pre-trained models (without valuehead)
     model = AutoModelForCausalLM.from_pretrained(
         model_to_load,
         config=config,
         torch_dtype=model_args.compute_dtype,
-        low_cpu_mem_usage=(not is_deepspeed_zero3_enabled()),
+        low_cpu_mem_usage=low_cpu_mem_usage,
         **config_kwargs
     )