diff --git a/tests/entrypoints/openai/test_exception_handlers.py b/tests/entrypoints/openai/test_exception_handlers.py new file mode 100644 index 000000000..d2b167829 --- /dev/null +++ b/tests/entrypoints/openai/test_exception_handlers.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Rejected requests are answered without re-raising into the ASGI server.""" + +from argparse import Namespace + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from vllm.entrypoints.openai.server_utils import register_exception_handlers +from vllm.exceptions import VLLMNotFoundError, VLLMValidationError + + +def _client() -> TestClient: + app = FastAPI() + app.state.args = Namespace(log_error_stack=False) + register_exception_handlers(app) + + @app.get("/raise/{kind}") + async def fail(kind: str): + if kind == "validation": + raise VLLMValidationError("prompt is longer than the maximum length") + if kind == "not_found": + raise VLLMNotFoundError("unknown model") + raise RuntimeError("server bug") + + # A re-raised exception surfaces here exactly as uvicorn would log it. + return TestClient(app, raise_server_exceptions=True) + + +@pytest.mark.parametrize(("kind", "status"), [("validation", 400), ("not_found", 404)]) +def test_client_errors_are_answered_without_a_traceback(kind, status): + response = _client().get(f"/raise/{kind}") + assert response.status_code == status + assert response.json()["error"]["code"] == status + + +def test_server_errors_still_reach_the_server_log(): + with pytest.raises(RuntimeError, match="server bug"): + _client().get("/raise/bug") diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 1632d4d06..241c146ec 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -16,8 +16,7 @@ from contextlib import asynccontextmanager from typing import Any import uvloop -from fastapi import FastAPI, HTTPException -from fastapi.exceptions import RequestValidationError +from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from starlette.datastructures import State @@ -29,18 +28,13 @@ from vllm.entrypoints.chat_utils import load_chat_template from vllm.entrypoints.launcher import serve_http from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args -from vllm.entrypoints.openai.engine.protocol import GenerationError from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.server_utils import ( - engine_error_handler, - exception_handler, - generation_error_handler, get_uvicorn_log_config, - http_exception_handler, lifespan, log_response, - validation_exception_handler, + register_exception_handlers, ) from vllm.entrypoints.sagemaker.api_router import sagemaker_standards_bootstrap from vllm.entrypoints.serve.elastic_ep.middleware import ( @@ -63,7 +57,6 @@ from vllm.usage.usage_lib import UsageContext from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.network_utils import is_valid_ipv6_address from vllm.utils.system_utils import decorate_logs, set_ulimit -from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError from vllm.version import __version__ as VLLM_VERSION prometheus_multiproc_dir: tempfile.TemporaryDirectory @@ -254,12 +247,7 @@ def build_app( allow_headers=args.allowed_headers, ) - app.exception_handler(HTTPException)(http_exception_handler) - app.exception_handler(RequestValidationError)(validation_exception_handler) - app.exception_handler(EngineGenerateError)(engine_error_handler) - app.exception_handler(EngineDeadError)(engine_error_handler) - app.exception_handler(GenerationError)(generation_error_handler) - app.exception_handler(Exception)(exception_handler) + register_exception_handlers(app) # Ensure --api-key option from CLI takes precedence over VLLM_API_KEY if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]: diff --git a/vllm/entrypoints/openai/server_utils.py b/vllm/entrypoints/openai/server_utils.py index 269c33549..fd54a91b3 100644 --- a/vllm/entrypoints/openai/server_utils.py +++ b/vllm/entrypoints/openai/server_utils.py @@ -27,7 +27,7 @@ from vllm.entrypoints.openai.engine.protocol import ( GenerationError, ) from vllm.entrypoints.utils import create_error_response, sanitize_message -from vllm.exceptions import VLLMValidationError +from vllm.exceptions import VLLMNotFoundError, VLLMValidationError from vllm.logger import init_logger from vllm.utils.gc_utils import freeze_gc_heap from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError @@ -388,6 +388,21 @@ async def exception_handler(req: Request, exc: Exception): return JSONResponse(err.model_dump(), status_code=err.error.code) +def register_exception_handlers(app: FastAPI) -> None: + app.exception_handler(HTTPException)(http_exception_handler) + app.exception_handler(RequestValidationError)(validation_exception_handler) + app.exception_handler(EngineGenerateError)(engine_error_handler) + app.exception_handler(EngineDeadError)(engine_error_handler) + app.exception_handler(GenerationError)(generation_error_handler) + # Starlette runs a handler registered for Exception in its + # ServerErrorMiddleware, which re-raises after responding: uvicorn then logs + # a full traceback for every rejected request. Client errors get their own + # registration so they are answered without being re-raised. + app.exception_handler(VLLMValidationError)(exception_handler) + app.exception_handler(VLLMNotFoundError)(exception_handler) + app.exception_handler(Exception)(exception_handler) + + async def http_exception_handler(req: Request, exc: HTTPException): if req.app.state.args.log_error_stack: logger.exception(