update patcher

Former-commit-id: d6d7b6670847ce4ea10353c5b126214542b45c2b
This commit is contained in:
hiyouga
2023-12-23 15:24:27 +08:00
parent f869e44fe5
commit 940403720a
6 changed files with 135 additions and 130 deletions

View File

@@ -22,6 +22,10 @@ class ModelArguments:
default=False,
metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}
)
resize_vocab: Optional[bool] = field(
default=False,
metadata={"help": "Whether to resize the tokenizer vocab and the embedding layers."}
)
split_special_tokens: Optional[bool] = field(
default=False,
metadata={"help": "Whether or not the special tokens should be split during the tokenization process."}