[config] update args (#7231)

Former-commit-id: f71a901840811bf560df671ec63a146ff99140c6
This commit is contained in:
hoshi-hiyouga
2025-03-10 23:04:43 +08:00
committed by GitHub
parent cf58a6d860
commit 71a1c1321a
16 changed files with 89 additions and 74 deletions

View File

@@ -21,6 +21,7 @@ from typing import Optional
from typing_extensions import Annotated
from ..chat import ChatModel
from ..extras.constants import EngineName
from ..extras.misc import torch_gc
from ..extras.packages import is_fastapi_available, is_starlette_available, is_uvicorn_available
from .chat import (
@@ -60,7 +61,7 @@ async def sweeper() -> None:
@asynccontextmanager
async def lifespan(app: "FastAPI", chat_model: "ChatModel"): # collects GPU memory
if chat_model.engine_type == "huggingface":
if chat_model.engine.name == EngineName.HF:
asyncio.create_task(sweeper())
yield
@@ -106,7 +107,7 @@ def create_app(chat_model: "ChatModel") -> "FastAPI":
if request.stream:
generate = create_stream_chat_completion_response(request, chat_model)
return EventSourceResponse(generate, media_type="text/event-stream")
return EventSourceResponse(generate, media_type="text/event-stream", sep="\n")
else:
return await create_chat_completion_response(request, chat_model)