ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support llama pro #2338 , add rslora

Browse Source 
Former-commit-id: 40d659b7f30dd5a004703c176ec1f22dc864e505
This commit is contained in:
hiyouga
2024-02-15 02:27:36 +08:00
parent b403f8d8a8
commit 596b6828cb
24 changed files with 438 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/llmtuner/hparams/data_args.py
View File

@@ -9,30 +9,40 @@ class DataArguments:
"""
template: Optional[str] = field(
default=None, metadata={"help": "Which template to use for constructing prompts in training and inference."}
default=None,
metadata={"help": "Which template to use for constructing prompts in training and inference."},
)
dataset: Optional[str] = field(
default=None,
metadata={"help": "The name of provided dataset(s) to use. Use commas to separate multiple datasets."},
)
dataset_dir: Optional[str] = field(
default="data", metadata={"help": "Path to the folder containing the datasets."}
default="data",
metadata={"help": "Path to the folder containing the datasets."},
)
split: Optional[str] = field(
default="train", metadata={"help": "Which dataset split to use for training and evaluation."}
default="train",
metadata={"help": "Which dataset split to use for training and evaluation."},
)
cutoff_len: Optional[int] = field(
default=1024, metadata={"help": "The cutoff length of the model inputs after tokenization."}
default=1024,
metadata={"help": "The cutoff length of the model inputs after tokenization."},
)
reserved_label_len: Optional[int] = field(
default=1, metadata={"help": "The minimum cutoff length reserved for label after tokenization."}
default=1,
metadata={"help": "The minimum cutoff length reserved for label after tokenization."},
)
train_on_prompt: Optional[bool] = field(
default=False, metadata={"help": "Whether to disable the mask on the prompt or not."}
default=False,
metadata={"help": "Whether to disable the mask on the prompt or not."},
)
streaming: Optional[bool] = field(
default=False,
metadata={"help": "Enable dataset streaming."},
)
streaming: Optional[bool] = field(default=False, metadata={"help": "Enable dataset streaming."})
buffer_size: Optional[int] = field(
default=16384, metadata={"help": "Size of the buffer to randomly sample examples from in dataset streaming."}
default=16384,
metadata={"help": "Size of the buffer to randomly sample examples from in dataset streaming."},
)
mix_strategy: Optional[Literal["concat", "interleave_under", "interleave_over"]] = field(
default="concat",
@@ -43,13 +53,16 @@ class DataArguments:
metadata={"help": "Probabilities to sample data from datasets. Use commas to separate multiple datasets."},
)
overwrite_cache: Optional[bool] = field(
default=False, metadata={"help": "Overwrite the cached training and evaluation sets."}
default=False,
metadata={"help": "Overwrite the cached training and evaluation sets."},
)
preprocessing_num_workers: Optional[int] = field(
default=None, metadata={"help": "The number of processes to use for the preprocessing."}
default=None,
metadata={"help": "The number of processes to use for the preprocessing."},
)
max_samples: Optional[int] = field(
default=None, metadata={"help": "For debugging purposes, truncate the number of examples for each dataset."}
default=None,
metadata={"help": "For debugging purposes, truncate the number of examples for each dataset."},
)
eval_num_beams: Optional[int] = field(
default=None,
@@ -62,13 +75,16 @@ class DataArguments:
},
)
val_size: Optional[float] = field(
default=0, metadata={"help": "Size of the development set, should be an integer or a float in range `[0,1)`."}
default=0,
metadata={"help": "Size of the development set, should be an integer or a float in range `[0,1)`."},
)
sft_packing: Optional[bool] = field(
default=False, metadata={"help": "Packing the questions and answers in the supervised fine-tuning stage."}
default=False,
metadata={"help": "Packing the questions and answers in the supervised fine-tuning stage."},
)
cache_path: Optional[str] = field(
default=None, metadata={"help": "Path to save or load the preprocessed datasets."}
default=None,
metadata={"help": "Path to save or load the preprocessed datasets."},
)
def __post_init__(self):