ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update webui

Browse Source 
Former-commit-id: da30d0fb4abdb825f3383ddd106bb06a84695b7a
This commit is contained in:
hiyouga
2023-08-14 22:45:26 +08:00
parent 5fc5e776ff
commit 7f0b908de2
8 changed files with 47 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/extras/misc.py
View File

@@ -95,7 +95,6 @@ def prepare_model_for_training(
use_gradient_checkpointing: Optional[bool] = True,
layer_norm_names: Optional[List[str]] = LAYERNORM_NAMES
) -> "PreTrainedModel":
for name, param in model.named_parameters():
if param.ndim == 1 and any(layer_norm_name in name for layer_norm_name in layer_norm_names):
param.data = param.data.to(torch.float32)
@@ -112,9 +111,6 @@ def prepare_model_for_training(
model.config.use_cache = False # turn off when gradient checkpointing is enabled
if finetuning_type != "full" and hasattr(model, output_layer_name):
if hasattr(model, "config") and hasattr(model.config, "pretraining_tp"):
model.config.pretraining_tp = 1 # disable TP for LoRA (https://github.com/huggingface/peft/pull/728)
output_layer: torch.nn.Linear = getattr(model, output_layer_name)
input_dtype = output_layer.weight.dtype