support image input in api #3971 #4061

Former-commit-id: c70aaf763ef22fb83ce3635e8ffd5ec4c89c1cb0
2024-06-06 02:29:55 +08:00
parent 35379c7c0e
commit 639a7f6796
4 changed files with 49 additions and 8 deletions
--- a/src/llamafactory/api/protocol.py
+++ b/src/llamafactory/api/protocol.py
@@ -56,9 +56,19 @@ class FunctionCall(BaseModel):
    function: Function


+class ImageURL(BaseModel):
+    url: str
+
+
+class MultimodalInputItem(BaseModel):
+    type: Literal["text", "image_url"]
+    text: Optional[str] = None
+    image_url: Optional[ImageURL] = None
+
+
 class ChatMessage(BaseModel):
    role: Role
-    content: Optional[str] = None
+    content: Optional[Union[str, List[MultimodalInputItem]]] = None
    tool_calls: Optional[List[FunctionCall]] = None