video datasets

Former-commit-id: 33f28ce82d9e44d2615909250dc56d6a4a03cd99

src/llamafactory/data/processors/unsupervised.py

@@ -24,7 +24,7 @@ if TYPE_CHECKING:
     from transformers import PreTrainedTokenizer, ProcessorMixin
 
     from ...hparams import DataArguments
-    from ..mm_plugin import ImageInput
+    from ..mm_plugin import ImageInput, VideoInput
     from ..template import Template
 
 
@@ -37,6 +37,7 @@ def _encode_unsupervised_example(
     system: Optional[str],
     tools: Optional[str],
     images: Sequence["ImageInput"],
+    videos: Sequence["VideoInput"],
     template: "Template",
     tokenizer: "PreTrainedTokenizer",
     processor: Optional["ProcessorMixin"],
@@ -47,12 +48,12 @@ def _encode_unsupervised_example(
     else:
         messages = prompt + [{"role": Role.ASSISTANT.value, "content": ""}]
 
-    messages = template.mm_plugin.process_messages(messages, images, processor)
+    messages = template.mm_plugin.process_messages(messages, images, videos, processor)
     input_ids, labels = template.encode_oneturn(tokenizer, messages, system, tools)
     if template.efficient_eos:
         labels += [tokenizer.eos_token_id]
 
-    input_ids, _ = template.mm_plugin.process_token_ids(input_ids, None, images, tokenizer, processor)
+    input_ids, _ = template.mm_plugin.process_token_ids(input_ids, None, images, videos, tokenizer, processor)
     source_len, target_len = infer_seqlen(len(input_ids), len(labels), cutoff_len)
     input_ids = input_ids[:source_len]
     labels = labels[:target_len]
@@ -79,6 +80,7 @@ def preprocess_unsupervised_dataset(
             system=examples["_system"][i],
             tools=examples["_tools"][i],
             images=examples["_images"][i] or [],
+            videos=examples["_videos"][i] or [],
             template=template,
             tokenizer=tokenizer,
             processor=processor,
@@ -88,6 +90,7 @@ def preprocess_unsupervised_dataset(
         model_inputs["attention_mask"].append([1] * len(input_ids))
         model_inputs["labels"].append(labels)
         model_inputs["images"].append(examples["_images"][i])
+        model_inputs["videos"].append(examples["_videos"][i])
 
     return model_inputs