Former-commit-id: 79731ae13ecd17eb8646fb53162c81dddfef3b00
This commit is contained in:
hoshi-hiyouga
2025-01-14 18:40:07 +08:00
committed by GitHub
parent 1bb06e06df
commit 41a9e231cb
5 changed files with 11 additions and 7 deletions

View File

@@ -85,7 +85,7 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule":
)
except Exception as e:
raise OSError("Failed to load tokenizer.") from e
if model_args.model_max_length is not None and tokenizer.model_max_length != model_args.model_max_length:
tokenizer.model_max_length = model_args.model_max_length