update parser

Former-commit-id: d98258aa08d93494ad50d7786064e7fda15f6ca9
This commit is contained in:
hiyouga
2024-03-10 13:35:20 +08:00
parent 7ff8a064f3
commit 7c492864e9
2 changed files with 29 additions and 19 deletions

View File

@@ -1,6 +1,6 @@
import json
import datasets
from typing import Any, Dict, List
from typing import Any, Dict, Generator, List, Tuple
_DESCRIPTION = "An example of dataset."
@@ -40,7 +40,7 @@ class ExampleDataset(datasets.GeneratorBasedBuilder):
)
]
def _generate_examples(self, filepath: str) -> Dict[int, Dict[str, Any]]:
def _generate_examples(self, filepath: str) -> Generator[Tuple[int, Dict[str, Any]], None, None]:
example_dataset = json.load(open(filepath, "r", encoding="utf-8"))
for key, example in enumerate(example_dataset):
yield key, example