ros/LlamaFactory
1
0
Fork 0
mirror of https://github.com/hiyouga/LlamaFactory.git synced 2026-02-01 20:23:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

1.Change the name of is_fastapi_available function

Browse Source 
2. Added the log of printing requests when deploying using vllm


Former-commit-id: 530d4f5d51c13c71d99de5fe2d23805b0aa875a2
This commit is contained in:
Tendo33
2024-05-09 14:28:01 +08:00
parent 4ce4172c87
commit 9dadff90bb
4 changed files with 64 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/llmtuner/api/app.py
View File

@@ -4,7 +4,7 @@ from typing import Annotated, Optional
from ..chat import ChatModel
from ..extras.misc import torch_gc
from ..extras.packages import is_fastapi_availble, is_starlette_available, is_uvicorn_available
from ..extras.packages import is_fastapi_available, is_starlette_available, is_uvicorn_available
from .chat import (
create_chat_completion_response,
create_score_evaluation_response,
@@ -20,7 +20,7 @@ from .protocol import (
)
if is_fastapi_availble():
if is_fastapi_available():
from fastapi import Depends, FastAPI, HTTPException, status
from fastapi.middleware.cors import CORSMiddleware
from fastapi.security.http import HTTPAuthorizationCredentials, HTTPBearer
@@ -54,7 +54,8 @@ def create_app(chat_model: "ChatModel") -> "FastAPI":
async def verify_api_key(auth: Annotated[Optional[HTTPAuthorizationCredentials], Depends(security)]):
if api_key and (auth is None or auth.credentials != api_key):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key.")
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key.")
@app.get(
"/v1/models",
@@ -74,10 +75,12 @@ def create_app(chat_model: "ChatModel") -> "FastAPI":
)
async def create_chat_completion(request: ChatCompletionRequest):
if not chat_model.engine.can_generate:
raise HTTPException(status_code=status.HTTP_405_METHOD_NOT_ALLOWED, detail="Not allowed")
raise HTTPException(
status_code=status.HTTP_405_METHOD_NOT_ALLOWED, detail="Not allowed")
if request.stream:
generate = create_stream_chat_completion_response(request, chat_model)
generate = create_stream_chat_completion_response(
request, chat_model)
return EventSourceResponse(generate, media_type="text/event-stream")
else:
return await create_chat_completion_response(request, chat_model)
@@ -90,7 +93,8 @@ def create_app(chat_model: "ChatModel") -> "FastAPI":
)
async def create_score_evaluation(request: ScoreEvaluationRequest):
if chat_model.engine.can_generate:
raise HTTPException(status_code=status.HTTP_405_METHOD_NOT_ALLOWED, detail="Not allowed")
raise HTTPException(
status_code=status.HTTP_405_METHOD_NOT_ALLOWED, detail="Not allowed")
return await create_score_evaluation_response(request, chat_model)