support system column #1765

Former-commit-id: f425584a511c5e42bae8b3ba090eaa898b28adad
2023-12-12 19:45:59 +08:00
parent c27675f70d
commit 934d00ea1e
14 changed files with 42 additions and 76 deletions
--- a/src/llmtuner/model/loader.py
+++ b/src/llmtuner/model/loader.py
@@ -217,7 +217,7 @@ def load_model_and_tokenizer(
    # Prepare model for inference
    if not is_trainable:
        model.requires_grad_(False) # fix all model params
-        model = model.to(model_args.compute_dtype) if model_args.quantization_bit is None else model
+        model = model.to(model_args.compute_dtype) if not getattr(model, "quantization_method", None) else model
        model.eval()
    else:
        model.train()