[parser] support omegaconf (#7793)

This commit is contained in:
hoshi-hiyouga
2025-04-21 23:30:30 +08:00
committed by GitHub
parent bd7bc31c79
commit 416853dd25
25 changed files with 62 additions and 94 deletions

View File

@@ -50,6 +50,10 @@ class DataCollatorWithVerbose(DataCollatorWithPadding):
verbose_list: list[dict[str, Any]] = field(default_factory=list)
def __call__(self, features: list[dict[str, Any]]) -> dict[str, Any]:
features = [
{k: v for k, v in feature.items() if k in ["input_ids", "attention_mask", "labels"]}
for feature in features
]
self.verbose_list.extend(features)
batch = super().__call__(features)
return {k: v[:, :1] for k, v in batch.items()} # truncate input length