ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reimplement neftune

Browse Source 
Former-commit-id: efe9e5a194d3a9f052701d904715238816e4c09e
This commit is contained in:
hiyouga
2023-10-22 16:15:08 +08:00
parent 1f69965239
commit 6da51565f5
9 changed files with 36 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/tuner/core/parser.py
View File

@@ -146,7 +146,7 @@ def get_train_args(
if not finetuning_args.resume_lora_training:
raise ValueError("Quantized model cannot create new LoRA weight. Merge them first.")
if training_args.do_train and model_args.quantization_bit is not None and (not model_args.upcast_layernorm):
if training_args.do_train and model_args.quantization_bit is not None and (not finetuning_args.upcast_layernorm):
logger.warning("We recommend enable `upcast_layernorm` in quantized training.")
if training_args.do_train and (not training_args.fp16) and (not training_args.bf16):