[config] update args (#7231)
Former-commit-id: f71a901840811bf560df671ec63a146ff99140c6
This commit is contained in:
@@ -21,6 +21,7 @@ from typing import Optional
|
||||
from typing_extensions import Annotated
|
||||
|
||||
from ..chat import ChatModel
|
||||
from ..extras.constants import EngineName
|
||||
from ..extras.misc import torch_gc
|
||||
from ..extras.packages import is_fastapi_available, is_starlette_available, is_uvicorn_available
|
||||
from .chat import (
|
||||
@@ -60,7 +61,7 @@ async def sweeper() -> None:
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: "FastAPI", chat_model: "ChatModel"): # collects GPU memory
|
||||
if chat_model.engine_type == "huggingface":
|
||||
if chat_model.engine.name == EngineName.HF:
|
||||
asyncio.create_task(sweeper())
|
||||
|
||||
yield
|
||||
@@ -106,7 +107,7 @@ def create_app(chat_model: "ChatModel") -> "FastAPI":
|
||||
|
||||
if request.stream:
|
||||
generate = create_stream_chat_completion_response(request, chat_model)
|
||||
return EventSourceResponse(generate, media_type="text/event-stream")
|
||||
return EventSourceResponse(generate, media_type="text/event-stream", sep="\n")
|
||||
else:
|
||||
return await create_chat_completion_response(request, chat_model)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user