Merge branch 'hiyouga:main' into main

Former-commit-id: 8619ad7dc124c50e254b1bb2e173ff99ca4f0e22
This commit is contained in:
Zhangchi Feng
2024-09-10 13:20:24 +08:00
committed by GitHub
5 changed files with 146 additions and 108 deletions

View File

@@ -357,6 +357,7 @@ def get_template_and_fix_tokenizer(tokenizer: "PreTrainedTokenizer", data_args:
require_version(
"transformers>=4.45.0.dev0", "To fix: pip install git+https://github.com/huggingface/transformers.git"
)
require_version("accelerate>=0.34.0", "To fix: pip install accelerate>=0.34.0")
if data_args.template is None:
template = TEMPLATES["empty"] # placeholder