ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[v1] add data converter (#9263)

Browse Source
This commit is contained in:
Yaowei Zheng
2025-10-13 15:54:47 +08:00
committed by GitHub
parent 48974783da
commit 52e46e162e
7 changed files with 266 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tests_v1/core/test_data_engine.py
View File

@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import random
import pytest
@@ -22,14 +21,11 @@ from llamafactory.v1.config.data_args import DataArguments
from llamafactory.v1.core.data_engine import DataEngine
TINY_DATA = os.getenv("TINY_DATA", "llamafactory/v1-sft-demo")
@pytest.mark.parametrize("num_samples", [16])
def test_map_dataset(num_samples: int):
data_args = DataArguments(dataset=TINY_DATA)
data_args = DataArguments(dataset="llamafactory/v1-sft-demo")
data_engine = DataEngine(data_args)
original_data = load_dataset(TINY_DATA, split="train")
original_data = load_dataset("llamafactory/v1-sft-demo", split="train")
indexes = random.choices(range(len(data_engine)), k=num_samples)
for index in indexes:
print(data_engine[index])