ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[data] fix loader (#7207)

Browse Source 
* fix dataloader

* add test case

* fix type

* fix ci

* fix ci

* fix ci

* disable overwrite cache in ci

Former-commit-id: e84af0e140b1aafd1a6d6fe185a8e41c8fc5f831
This commit is contained in:
hoshi-hiyouga
2025-03-07 17:20:46 +08:00
committed by GitHub
parent 82a2bac866
commit 16419b2834
16 changed files with 161 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tests/data/processor/test_pairwise.py
View File

@@ -21,7 +21,7 @@ from datasets import load_dataset
from transformers import AutoTokenizer
from llamafactory.extras.constants import IGNORE_INDEX
from llamafactory.train.test_utils import load_train_dataset
from llamafactory.train.test_utils import load_dataset_module
DEMO_DATA = os.getenv("DEMO_DATA", "llamafactory/demo_data")
@@ -37,7 +37,6 @@ TRAIN_ARGS = {
"dataset_dir": "REMOTE:" + DEMO_DATA,
"template": "llama3",
"cutoff_len": 8192,
"overwrite_cache": True,
"output_dir": "dummy_dir",
"overwrite_output_dir": True,
"fp16": True,
@@ -55,7 +54,7 @@ def _convert_sharegpt_to_openai(messages: List[Dict[str, str]]) -> List[Dict[str
@pytest.mark.parametrize("num_samples", [16])
def test_pairwise_data(num_samples: int):
train_dataset = load_train_dataset(**TRAIN_ARGS)
train_dataset = load_dataset_module(**TRAIN_ARGS)["train_dataset"]
ref_tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA)
original_data = load_dataset(DEMO_DATA, name="dpo_en_demo", split="train")
indexes = random.choices(range(len(original_data)), k=num_samples)