ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix rope scaling

Browse Source 
Former-commit-id: 2e0dd36700ec5e8294581c1db4b9431f755fc5f8
This commit is contained in:
hiyouga
2023-08-12 22:00:01 +08:00
parent fc48bd8da0
commit c95479dddb
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/llmtuner/tuner/core/loader.py
View File

@@ -102,11 +102,11 @@ def load_model_and_tokenizer(
) )
current_max_length = getattr(config, "max_position_embeddings", None) current_max_length = getattr(config, "max_position_embeddings", None)
if current_max_length and model_args.model_max_length <= current_max_length: if current_max_length and model_args.model_max_length > current_max_length:
scaling_factor = float(math.ceil(model_args.model_max_length / current_max_length))
else:
logger.warning("Input length is smaller than max length. Consider increase input length.") logger.warning("Input length is smaller than max length. Consider increase input length.")
scaling_factor = 1.0 scaling_factor = 1.0
else:
scaling_factor = float(math.ceil(model_args.model_max_length / current_max_length))
else: else:
scaling_factor = 2.0 scaling_factor = 2.0