Add support for Index

Former-commit-id: 4e6dba16ca1755235d2ae117b53b68c5ae2f239a
2024-11-02 13:45:27 +08:00
parent 83479c9ef0
commit dbaf621f57
4 changed files with 39 additions and 2 deletions
--- a/src/llamafactory/data/template.py
+++ b/src/llamafactory/data/template.py
@@ -691,6 +691,14 @@ _register_template(
 )


+_register_template(
+    name="index",
+    format_user=StringFormatter(slots=["reserved_0{{content}}reserved_1"]),
+    format_system=StringFormatter(slots=["<unk>{{content}}"]),
+    efficient_eos=True,
+)
+
+
 _register_template(
    name="intern",
    format_user=StringFormatter(slots=["<|User|>:{{content}}\n<|Bot|>:"]),