support quantization in export model

Former-commit-id: f32500ae6edccab7d14df4c92467e15986866def
2023-12-15 23:44:50 +08:00
parent 9121722999
commit 296711d502
9 changed files with 120 additions and 32 deletions
--- a/src/llmtuner/data/utils.py
+++ b/src/llmtuner/data/utils.py
@@ -12,16 +12,6 @@ if TYPE_CHECKING:
 logger = get_logger(__name__)


-EXT2TYPE = {
-    "arrow": "arrow",
-    "csv": "csv",
-    "json": "json",
-    "jsonl": "json",
-    "parquet": "parquet",
-    "txt": "text"
-}
-
-
 def checksum(data_files: List[str], file_sha1: Optional[str] = None) -> None:
    if file_sha1 is None:
        logger.warning("Checksum failed: missing SHA-1 hash value in dataset_info.json.")