update patcher
Former-commit-id: d6d7b6670847ce4ea10353c5b126214542b45c2b
This commit is contained in:
@@ -22,6 +22,10 @@ class ModelArguments:
|
||||
default=False,
|
||||
metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}
|
||||
)
|
||||
resize_vocab: Optional[bool] = field(
|
||||
default=False,
|
||||
metadata={"help": "Whether to resize the tokenizer vocab and the embedding layers."}
|
||||
)
|
||||
split_special_tokens: Optional[bool] = field(
|
||||
default=False,
|
||||
metadata={"help": "Whether or not the special tokens should be split during the tokenization process."}
|
||||
|
||||
Reference in New Issue
Block a user