ros/LlamaFactory
1
0
Fork 0
mirror of https://github.com/hiyouga/LlamaFactory.git synced 2026-02-03 21:03:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

[deps] adapt to transformers v5 (#10147)

Browse Source 
Co-authored-by: frozenleaves <frozen@Mac.local>
Co-authored-by: hiyouga <hiyouga@buaa.edu.cn>
This commit is contained in:
浮梦
2026-02-02 12:07:19 +08:00
committed by GitHub
parent 762b480131
commit bf04ca6af8
23 changed files with 149 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tests/data/processor/test_feedback.py
View File

@@ -20,6 +20,7 @@ from datasets import load_dataset
from transformers import AutoTokenizer
from llamafactory.extras.constants import IGNORE_INDEX
from llamafactory.extras.packages import is_transformers_version_greater_than
from llamafactory.train.test_utils import load_dataset_module
@@ -52,7 +53,12 @@ def test_feedback_data(num_samples: int):
for index in indexes:
messages = original_data["messages"][index]
ref_input_ids = ref_tokenizer.apply_chat_template(messages)
prompt_len = len(ref_tokenizer.apply_chat_template(messages[:-1], add_generation_prompt=True))
ref_prompt_ids = ref_tokenizer.apply_chat_template(messages[:-1], add_generation_prompt=True)
if is_transformers_version_greater_than("5.0.0"):
ref_input_ids = ref_input_ids["input_ids"]
ref_prompt_ids = ref_prompt_ids["input_ids"]
prompt_len = len(ref_prompt_ids)
ref_labels = [IGNORE_INDEX] * prompt_len + ref_input_ids[prompt_len:]
assert train_dataset["input_ids"][index] == ref_input_ids
assert train_dataset["labels"][index] == ref_labels

19
tests/data/processor/test_pairwise.py
View File

@@ -20,6 +20,7 @@ from datasets import load_dataset
from transformers import AutoTokenizer
from llamafactory.extras.constants import IGNORE_INDEX
from llamafactory.extras.packages import is_transformers_version_greater_than
from llamafactory.train.test_utils import load_dataset_module
@@ -63,13 +64,21 @@ def test_pairwise_data(num_samples: int):
rejected_messages = original_data["conversations"][index] + [original_data["rejected"][index]]
chosen_messages = _convert_sharegpt_to_openai(chosen_messages)
rejected_messages = _convert_sharegpt_to_openai(rejected_messages)
ref_chosen_input_ids = ref_tokenizer.apply_chat_template(chosen_messages)
chosen_prompt_len = len(ref_tokenizer.apply_chat_template(chosen_messages[:-1], add_generation_prompt=True))
ref_chosen_labels = [IGNORE_INDEX] * chosen_prompt_len + ref_chosen_input_ids[chosen_prompt_len:]
ref_chosen_prompt_ids = ref_tokenizer.apply_chat_template(chosen_messages[:-1], add_generation_prompt=True)
ref_rejected_input_ids = ref_tokenizer.apply_chat_template(rejected_messages)
rejected_prompt_len = len(
ref_tokenizer.apply_chat_template(rejected_messages[:-1], add_generation_prompt=True)
)
ref_rejected_prompt_ids = ref_tokenizer.apply_chat_template(rejected_messages[:-1], add_generation_prompt=True)
if is_transformers_version_greater_than("5.0.0"):
ref_chosen_input_ids = ref_chosen_input_ids["input_ids"]
ref_rejected_input_ids = ref_rejected_input_ids["input_ids"]
ref_chosen_prompt_ids = ref_chosen_prompt_ids["input_ids"]
ref_rejected_prompt_ids = ref_rejected_prompt_ids["input_ids"]
chosen_prompt_len = len(ref_chosen_prompt_ids)
rejected_prompt_len = len(ref_rejected_prompt_ids)
ref_chosen_labels = [IGNORE_INDEX] * chosen_prompt_len + ref_chosen_input_ids[chosen_prompt_len:]
ref_rejected_labels = [IGNORE_INDEX] * rejected_prompt_len + ref_rejected_input_ids[rejected_prompt_len:]
assert train_dataset["chosen_input_ids"][index] == ref_chosen_input_ids
assert train_dataset["chosen_labels"][index] == ref_chosen_labels

31
tests/data/processor/test_supervised.py
View File

@@ -20,6 +20,7 @@ from datasets import load_dataset
from transformers import AutoTokenizer
from llamafactory.extras.constants import IGNORE_INDEX
from llamafactory.extras.packages import is_transformers_version_greater_than
from llamafactory.train.test_utils import load_dataset_module
@@ -59,7 +60,16 @@ def test_supervised_single_turn(num_samples: int):
{"role": "assistant", "content": original_data["output"][index]},
]
ref_input_ids = ref_tokenizer.apply_chat_template(messages)
ref_prompt_ids = ref_tokenizer.apply_chat_template(messages[:-1], add_generation_prompt=True)
if is_transformers_version_greater_than("5.0.0"):
ref_input_ids = ref_input_ids["input_ids"]
ref_prompt_ids = ref_prompt_ids["input_ids"]
prompt_len = len(ref_prompt_ids)
ref_label_ids = [IGNORE_INDEX] * prompt_len + ref_input_ids[prompt_len:]
assert train_dataset["input_ids"][index] == ref_input_ids
assert train_dataset["labels"][index] == ref_label_ids
@pytest.mark.runs_on(["cpu", "mps"])
@@ -73,6 +83,10 @@ def test_supervised_multi_turn(num_samples: int):
indexes = random.choices(range(len(original_data)), k=num_samples)
for index in indexes:
ref_input_ids = ref_tokenizer.apply_chat_template(original_data["messages"][index])
if is_transformers_version_greater_than("5.0.0"):
ref_input_ids = ref_input_ids["input_ids"]
# cannot test the label ids in multi-turn case
assert train_dataset["input_ids"][index] == ref_input_ids
@@ -86,9 +100,12 @@ def test_supervised_train_on_prompt(num_samples: int):
original_data = load_dataset(DEMO_DATA, name="system_chat", split="train")
indexes = random.choices(range(len(original_data)), k=num_samples)
for index in indexes:
ref_ids = ref_tokenizer.apply_chat_template(original_data["messages"][index])
assert train_dataset["input_ids"][index] == ref_ids
assert train_dataset["labels"][index] == ref_ids
ref_input_ids = ref_tokenizer.apply_chat_template(original_data["messages"][index])
if is_transformers_version_greater_than("5.0.0"):
ref_input_ids = ref_input_ids["input_ids"]
assert train_dataset["input_ids"][index] == ref_input_ids
assert train_dataset["labels"][index] == ref_input_ids
@pytest.mark.runs_on(["cpu", "mps"])
@@ -103,7 +120,13 @@ def test_supervised_mask_history(num_samples: int):
for index in indexes:
messages = original_data["messages"][index]
ref_input_ids = ref_tokenizer.apply_chat_template(messages)
prompt_len = len(ref_tokenizer.apply_chat_template(messages[:-1], add_generation_prompt=True))
ref_prompt_ids = ref_tokenizer.apply_chat_template(messages[:-1], add_generation_prompt=True)
if is_transformers_version_greater_than("5.0.0"):
ref_input_ids = ref_input_ids["input_ids"]
ref_prompt_ids = ref_prompt_ids["input_ids"]
prompt_len = len(ref_prompt_ids)
ref_label_ids = [IGNORE_INDEX] * prompt_len + ref_input_ids[prompt_len:]
assert train_dataset["input_ids"][index] == ref_input_ids
assert train_dataset["labels"][index] == ref_label_ids

14
tests/data/processor/test_unsupervised.py
View File

@@ -19,6 +19,7 @@ import pytest
from datasets import load_dataset
from transformers import AutoTokenizer
from llamafactory.extras.packages import is_transformers_version_greater_than
from llamafactory.train.test_utils import load_dataset_module
@@ -55,8 +56,13 @@ def test_unsupervised_data(num_samples: int):
indexes = random.choices(range(len(original_data)), k=num_samples)
for index in indexes:
messages = original_data["messages"][index]
ref_ids = ref_tokenizer.apply_chat_template(messages)
ref_input_ids = ref_tokenizer.apply_chat_template(messages[:-1], add_generation_prompt=True)
ref_labels = ref_ids[len(ref_input_ids) :]
assert train_dataset["input_ids"][index] == ref_input_ids
ref_input_ids = ref_tokenizer.apply_chat_template(messages)
ref_prompt_ids = ref_tokenizer.apply_chat_template(messages[:-1], add_generation_prompt=True)
if is_transformers_version_greater_than("5.0.0"):
ref_input_ids = ref_input_ids["input_ids"]
ref_prompt_ids = ref_prompt_ids["input_ids"]
ref_labels = ref_input_ids[len(ref_prompt_ids) :]
assert train_dataset["input_ids"][index] == ref_prompt_ids
assert train_dataset["labels"][index] == ref_labels