remove dummy code

Former-commit-id: e6bc89d280945bbf48281107145c40a41d7cbd56
2023-05-30 16:28:00 +08:00
parent b6ed5176e1
commit 181c776b58
1 changed files with 0 additions and 3 deletions
--- a/src/utils/common.py
+++ b/src/utils/common.py
@@ -316,9 +316,6 @@ def prepare_data(
            max_samples_temp = min(len(dataset), max_samples)
            dataset = dataset.select(range(max_samples_temp))

-        if dataset.column_names[0] == "text": # for plaintext (in pre-training)
-            dataset = dataset.rename_column("text", getattr(dataset_attr, "prompt_column"))
-
        dummy_data = [None] * len(dataset)
        for column_name, target_name in [
            ("prompt_column", "prompt"),