ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix #6499

Browse Source 
Former-commit-id: dffc607220ff6dac15cf501ac9a3cdbe80c25211
This commit is contained in:
hiyouga
2025-01-02 11:17:29 +00:00
parent 5504b5254c
commit 2aaf3697d7
5 changed files with 17 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/llamafactory/data/processors/pretrain.py
View File

@@ -52,3 +52,8 @@ def preprocess_pretrain_dataset(
result["input_ids"][i][0] = tokenizer.bos_token_id
return result
def print_pretrain_dataset_example(example: Dict[str, List[int]], tokenizer: "PreTrainedTokenizer") -> None:
print("input_ids:\n{}".format(example["input_ids"]))
print("inputs:\n{}".format(tokenizer.decode(example["input_ids"], skip_special_tokens=False)))

2
src/llamafactory/data/processors/unsupervised.py
View File

@@ -100,3 +100,5 @@ def preprocess_unsupervised_dataset(
def print_unsupervised_dataset_example(example: Dict[str, List[int]], tokenizer: "PreTrainedTokenizer") -> None:
print("input_ids:\n{}".format(example["input_ids"]))
print("inputs:\n{}".format(tokenizer.decode(example["input_ids"], skip_special_tokens=False)))
print("label_ids:\n{}".format(example["labels"]))
print("labels:\n{}".format(tokenizer.decode(example["labels"], skip_special_tokens=False)))