Former-commit-id: 830511a6d0216da99520aee8b3a753d347a71fa9
This commit is contained in:
hiyouga
2024-08-30 03:21:50 +08:00
parent 30d97ca879
commit 665db18661
8 changed files with 24 additions and 13 deletions

View File

@@ -74,6 +74,9 @@ class BasePlugin:
images: Sequence["ImageObject"],
processor: Optional["ProcessorMixin"],
) -> List[Dict[str, str]]:
r"""
Pre-processes input messages before tokenization for VLMs.
"""
return messages
def process_token_ids(
@@ -83,6 +86,9 @@ class BasePlugin:
tokenizer: "PreTrainedTokenizer",
processor: Optional["ProcessorMixin"],
) -> Tuple[List[int], Optional[List[int]]]:
r"""
Pre-processes token ids after tokenization for VLMs.
"""
return input_ids, labels
def get_mm_inputs(
@@ -91,6 +97,9 @@ class BasePlugin:
feature_seqlens: Dict[str, int],
processor: Optional["ProcessorMixin"],
) -> Dict[str, Any]:
r"""
Builds batched multimodal inputs for VLMs.
"""
return {}
def process_model_inputs(
@@ -100,6 +109,9 @@ class BasePlugin:
feature_seqlens: Dict[str, int],
processor: Optional["ProcessorMixin"],
) -> None:
r"""
Appends multimodal inputs to model inputs for VLMs.
"""
return