Coverage for .venv/lib/python3.13/site-packages/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py: 37%
1261 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-10 12:01 +0000
1"""
2What is this?
4Provider-specific Pass-Through Endpoints
6Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc.
7"""
9from __future__ import annotations
11import hmac
12import inspect
13import json
14import os
15import posixpath
16import re
17import sys
18from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
19from dataclasses import dataclass
20from functools import partial
21from types import MappingProxyType
22from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast
24import httpx
25from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket
26from fastapi.responses import StreamingResponse
27from starlette.websockets import WebSocketState
28from typing_extensions import ReadOnly, TypedDict
30import litellm
31from litellm import get_llm_provider
32from litellm._logging import verbose_proxy_logger
33from litellm.constants import (
34 ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS,
35 AZURE_SPEECH_BATCH_PATH_PREFIX,
36 AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN,
37 AZURE_SPEECH_CUSTOM_LLM_PROVIDER,
38 AZURE_SPEECH_FAST_TRANSCRIPTION_PATH,
39 AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX,
40 AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX,
41 AZURE_SPEECH_STT_DOMAIN,
42 AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER,
43 BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES,
44)
45from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
46from litellm.llms.anthropic.common_utils import AnthropicModelInfo
47from litellm.llms.azure.passthrough.transformation import foreign_azure_deployment
48from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
49from litellm.llms.deepgram.common_utils import (
50 deepgram_listen_callback_params,
51 deepgram_listen_is_priced,
52 deepgram_listen_registry_key,
53 deepgram_listen_requested_model,
54 deepgram_listen_websocket_target,
55)
56from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base
57from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
58from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
59from litellm.passthrough.main import AsyncPassthroughStreamingResponse
60from litellm.proxy._types import *
61from litellm.proxy.auth.auth_checks import enforced_model_allowlists
62from litellm.proxy.auth.handle_jwt import JWTHandler
63from litellm.proxy.auth.route_checks import RouteChecks
64from litellm.proxy.auth.user_api_key_auth import (
65 _get_bearer_token,
66 is_no_auth_dev_mode,
67 user_api_key_auth,
68 user_api_key_auth_websocket,
69 user_api_key_auth_websocket_for_model,
70)
71from litellm.proxy.common_request_processing import open_sse_before_first_byte
72from litellm.proxy.common_utils.http_parsing_utils import (
73 _read_request_body,
74 _safe_get_request_headers,
75 _safe_set_request_parsed_body,
76 get_form_data,
77 get_request_body,
78 is_json_content_type,
79)
80from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
81from litellm.proxy.common_utils.sse_keepalive import (
82 wrap_passthrough_sse_bytes_with_keepalive_pings,
83)
84from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key
85from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
86 HttpPassThroughEndpointHelpers,
87 create_pass_through_route,
88 create_websocket_passthrough_route,
89 websocket_passthrough_request,
90)
91from litellm.proxy.utils import ProxyLogging as ProxyLoggingType
92from litellm.proxy.utils import is_known_model
93from litellm.proxy.vector_store_endpoints.utils import (
94 assert_proxy_admin_for_vector_store_index_management,
95 assert_user_can_access_vector_store,
96 get_litellm_managed_vector_store,
97 is_allowed_to_call_vector_store_endpoint,
98)
99from litellm.secret_managers.main import get_secret_str, str_to_bool
100from litellm.types.passthrough_endpoints.pass_through_endpoints import (
101 LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
102 LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
103 LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
104)
105from litellm.types.passthrough_endpoints.tinyfish import (
106 TINYFISH_AUTHENTICATED_RUN_FIELDS,
107 TINYFISH_PASSTHROUGH_TIMEOUT_SECONDS,
108 TINYFISH_REJECTED_ENVELOPE_FIELDS,
109 is_allowed_tinyfish_endpoint,
110)
111from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials
112from litellm.types.router import LiteLLMParamsTypedDict
113from litellm.types.utils import LlmProviders
114from litellm.types.vector_stores import LiteLLM_ManagedVectorStore
115from litellm.utils import ProviderConfigManager
117from .passthrough_endpoint_router import PassthroughEndpointRouter
119if TYPE_CHECKING: 119 ↛ 120line 119 didn't jump to line 120 because the condition on line 119 was never true
120 from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig
121 from litellm.router import Router
123 ProxyConfig = _ProxyConfig # rebind-ok: conditional type alias
124else:
125 ProxyConfig = Any # rebind-ok: runtime fallback
127vertex_llm_base: Final = VertexBase()
128router: Final = APIRouter()
129default_vertex_config: Final = None
130passthrough_endpoint_router: Final = PassthroughEndpointRouter()
133def create_request_copy(request: Request):
134 return {
135 "method": request.method,
136 "url": str(request.url),
137 "headers": _safe_get_request_headers(request).copy(),
138 "cookies": request.cookies,
139 "query_params": dict(request.query_params),
140 }
143def is_passthrough_request_using_router_model(request_body: dict, llm_router: litellm.Router | None) -> bool:
144 """
145 Returns True if the model is in the llm_router model names
146 """
147 try:
148 model: Final = request_body.get("model")
149 return is_known_model(model, llm_router)
150 except Exception:
151 return False
154class RelayRejection(TypedDict):
155 error: ReadOnly[str]
158def _deployment_model_name(litellm_params: LiteLLMParamsTypedDict) -> str:
159 model: Final = litellm_params.get("model", "")
160 try:
161 return get_llm_provider(model=model, custom_llm_provider=litellm_params.get("custom_llm_provider"))[0]
162 except litellm.BadRequestError:
163 return model
166def _models_served_by_group(llm_router: litellm.Router, model_group: str) -> frozenset[str]:
167 return frozenset(
168 _deployment_model_name(row["litellm_params"]) for row in llm_router.get_model_list(model_name=model_group) or ()
169 )
172def is_passthrough_request_streaming(request_body: object) -> bool:
173 """
174 Returns True if the request is streaming.
176 A JSON body need not be an object, so a list or scalar can reach here; it
177 carries no streaming flag.
178 """
179 if not isinstance(request_body, dict): 179 ↛ 180line 179 didn't jump to line 180 because the condition on line 179 was never true
180 return False
181 return bool(request_body.get("stream", False))
184def _optional_str(value: object) -> str | None:
185 return value if isinstance(value, str) else None
188def _string_keyed_mapping(value: object) -> Mapping[str, object] | None:
189 if isinstance(value, Mapping):
190 return value
191 return None
194async def _json_request_body(request: Request) -> Mapping[str, object]:
195 return await request.json()
198def get_passthrough_router_request_metadata(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, object]:
199 """
200 Build the request metadata carrying key-level spend attribution and the
201 pre-call budget reservation for a router-model passthrough request.
203 Router-model passthrough branches call ``allm_passthrough_route`` directly,
204 bypassing ``add_litellm_data_to_request``. Without this metadata the cost
205 callback cannot attribute spend to the calling key and never releases the
206 budget reservation minted at auth time, so the shared spend counter drifts
207 up until the key falsely trips ``BudgetExceededError``.
209 The payload rides the ``litellm_metadata`` bucket, not ``metadata``: the
210 router hop ``_ageneric_api_call_with_fallbacks`` canonicalises this call
211 type into ``litellm_metadata``, and the cost callback reads spend
212 attribution from that bucket while only backfilling ``user_api_key*`` keys
213 from ``metadata``. Passing ``metadata=`` would silently drop the secondary
214 attribution fields the helper sets (``agent_id``,
215 ``user_api_end_user_max_budget``) before the callback ever sees them.
216 """
217 from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
219 request_data: Final = {"litellm_metadata": {}} # mutable-ok: builder + litellm mutate this in place
220 LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
221 data=request_data,
222 user_api_key_dict=user_api_key_dict,
223 _metadata_variable_name="litellm_metadata",
224 )
225 return request_data["litellm_metadata"]
228async def llm_passthrough_factory_proxy_route(
229 custom_llm_provider: str,
230 endpoint: str,
231 request: Request,
232 fastapi_response: Response,
233 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
234):
235 """
236 Factory function for creating pass-through endpoints for LLM providers.
237 """
238 from litellm.types.utils import LlmProviders
239 from litellm.utils import ProviderConfigManager
241 provider_config: Final = ProviderConfigManager.get_provider_model_info(
242 provider=LlmProviders(custom_llm_provider),
243 model=None,
244 )
245 if provider_config is None: 245 ↛ 246line 245 didn't jump to line 246 because the condition on line 245 was never true
246 raise HTTPException(status_code=404, detail=f"Provider {custom_llm_provider} not found")
248 base_target_url: Final = provider_config.get_api_base()
250 if base_target_url is None:
251 raise HTTPException(status_code=404, detail=f"Provider {custom_llm_provider} api base not found")
253 encoded_endpoint = httpx.URL(endpoint).path
255 # Ensure endpoint starts with '/' for proper URL construction
256 if not encoded_endpoint.startswith("/"):
257 encoded_endpoint = "/" + encoded_endpoint
259 # Construct the full target URL using httpx, preserving any base path
260 # prefix that the operator configured on base_target_url.
261 base_url: Final = httpx.URL(base_target_url)
262 updated_url: Final = base_url.copy_with(
263 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
264 )
266 # Add or update query parameters
267 provider_api_key: Final = passthrough_endpoint_router.get_credentials(
268 custom_llm_provider=custom_llm_provider,
269 region_name=None,
270 )
272 auth_headers: Final = provider_config.validate_environment(
273 headers={},
274 model="",
275 messages=[],
276 optional_params={},
277 litellm_params={},
278 api_key=provider_api_key,
279 api_base=base_target_url,
280 )
282 ## check for streaming
283 is_streaming_request = False
284 # anthropic is streaming when 'stream' = True is in the body
285 if request.method == "POST":
286 if "multipart/form-data" not in request.headers.get("content-type", ""):
287 _request_body = await _json_request_body(request)
288 else:
289 _request_body = await get_form_data(request)
291 if _request_body.get("stream"):
292 is_streaming_request = True
294 ## CREATE PASS-THROUGH
295 endpoint_func: Final = create_pass_through_route(
296 endpoint=endpoint,
297 target=str(updated_url),
298 custom_headers=auth_headers,
299 is_streaming_request=is_streaming_request,
300 ) # dynamically construct pass-through endpoint based on incoming path
301 received_value: Final = await endpoint_func(
302 request,
303 fastapi_response,
304 user_api_key_dict,
305 )
307 return received_value
310@router.api_route(
311 "/gemini/{endpoint:path}",
312 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
313 tags=["Google AI Studio Pass-through", "pass-through"],
314)
315async def gemini_proxy_route(
316 endpoint: str,
317 request: Request,
318 fastapi_response: Response,
319):
320 """
321 [Docs](https://docs.litellm.ai/docs/pass_through/google_ai_studio)
322 """
323 ## CHECK FOR LITELLM API KEY IN THE QUERY PARAMS - ?..key=LITELLM_API_KEY
324 google_ai_studio_api_key: Final = request.query_params.get("key") or request.headers.get("x-goog-api-key")
326 user_api_key_dict: Final = await user_api_key_auth(request=request, api_key=f"Bearer {google_ai_studio_api_key}")
328 base_target_url: Final = os.getenv("GEMINI_API_BASE") or "https://generativelanguage.googleapis.com"
329 encoded_endpoint = httpx.URL(endpoint).path
331 # Ensure endpoint starts with '/' for proper URL construction
332 if not encoded_endpoint.startswith("/"):
333 encoded_endpoint = "/" + encoded_endpoint
335 # Construct the full target URL using httpx, preserving any base path
336 # prefix that the operator configured on base_target_url.
337 base_url: Final = httpx.URL(base_target_url)
338 updated_url: Final = base_url.copy_with(
339 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
340 )
342 # Add or update query parameters
343 gemini_api_key: Final[str | None] = passthrough_endpoint_router.get_credentials(
344 custom_llm_provider="gemini",
345 region_name=None,
346 )
347 if gemini_api_key is None:
348 raise Exception(
349 "Required 'GEMINI_API_KEY'/'GOOGLE_API_KEY' in environment to make pass-through calls to Google AI Studio."
350 )
351 # Merge query parameters, giving precedence to those in updated_url
352 merged_params: Final = dict(request.query_params)
353 merged_params.update({"key": gemini_api_key})
355 ## check for streaming
356 is_streaming_request = False
357 if "stream" in str(updated_url):
358 is_streaming_request = True
360 ## CREATE PASS-THROUGH
361 endpoint_func: Final = create_pass_through_route(
362 endpoint=endpoint,
363 target=str(updated_url),
364 custom_llm_provider="gemini",
365 is_streaming_request=is_streaming_request,
366 query_params=merged_params,
367 ) # dynamically construct pass-through endpoint based on incoming path
368 received_value: Final = await endpoint_func(
369 request,
370 fastapi_response,
371 user_api_key_dict,
372 )
374 return received_value
377@router.api_route(
378 "/cohere/{endpoint:path}",
379 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
380 tags=["Cohere Pass-through", "pass-through"],
381)
382async def cohere_proxy_route(
383 endpoint: str,
384 request: Request,
385 fastapi_response: Response,
386 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
387):
388 """
389 [Docs](https://docs.litellm.ai/docs/pass_through/cohere)
390 """
391 base_target_url: Final = os.getenv("COHERE_API_BASE") or "https://api.cohere.com"
392 encoded_endpoint = httpx.URL(endpoint).path
394 # Ensure endpoint starts with '/' for proper URL construction
395 if not encoded_endpoint.startswith("/"):
396 encoded_endpoint = "/" + encoded_endpoint
398 # Construct the full target URL using httpx, preserving any base path
399 # prefix that the operator configured on base_target_url.
400 base_url: Final = httpx.URL(base_target_url)
401 updated_url: Final = base_url.copy_with(
402 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
403 )
405 # Add or update query parameters
406 cohere_api_key: Final = passthrough_endpoint_router.get_credentials(
407 custom_llm_provider="cohere",
408 region_name=None,
409 )
411 ## check for streaming
412 is_streaming_request = False
413 if "stream" in str(updated_url): 413 ↛ 414line 413 didn't jump to line 414 because the condition on line 413 was never true
414 is_streaming_request = True
416 ## CREATE PASS-THROUGH
417 endpoint_func: Final = create_pass_through_route(
418 endpoint=endpoint,
419 target=str(updated_url),
420 custom_headers={"Authorization": f"Bearer {cohere_api_key}"},
421 is_streaming_request=is_streaming_request,
422 ) # dynamically construct pass-through endpoint based on incoming path
423 received_value: Final = await endpoint_func(
424 request,
425 fastapi_response,
426 user_api_key_dict,
427 )
429 return received_value
432def _fal_target(endpoint: str) -> httpx.URL:
433 base_target_url: Final = fal_ai_queue_base()
434 encoded_endpoint: Final = httpx.URL(endpoint).path
435 normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}"
436 base_url: Final = httpx.URL(base_target_url)
437 return base_url.copy_with(
438 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint),
439 )
442@router.api_route(
443 "/fal_ai/{endpoint:path}",
444 methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list
445 tags=["Fal AI Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list
446)
447async def fal_ai_proxy_route(
448 endpoint: str,
449 request: Request,
450 fastapi_response: Response,
451 user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
452):
453 updated_url: Final = _fal_target(endpoint)
454 fal_ai_api_key: Final = passthrough_endpoint_router.get_credentials(
455 custom_llm_provider="fal_ai",
456 region_name=None,
457 )
458 if fal_ai_api_key is None: 458 ↛ 463line 458 didn't jump to line 463 because the condition on line 458 was always true
459 raise HTTPException(
460 status_code=401,
461 detail="FAL_AI_API_KEY is not set and no fal_ai pass-through deployment credentials are configured",
462 )
463 if "/requests/" not in endpoint and fal_ai_passthrough_cost(endpoint, await _read_request_body(request)) is None:
464 raise HTTPException(
465 status_code=400,
466 detail=f"fal_ai/{endpoint} has no pricing entry for this request; only priced Fal requests can be submitted through /fal_ai",
467 )
468 endpoint_func: Final = create_pass_through_route(
469 endpoint=endpoint,
470 target=str(updated_url),
471 custom_headers={"Authorization": f"Key {fal_ai_api_key}"},
472 custom_llm_provider="fal_ai",
473 is_streaming_request=False,
474 )
475 return await endpoint_func(request, fastapi_response, user_api_key_dict)
478@router.api_route(
479 "/vllm/{endpoint:path}",
480 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
481 tags=["VLLM Pass-through", "pass-through"],
482)
483async def vllm_proxy_route(
484 endpoint: str,
485 request: Request,
486 fastapi_response: Response,
487 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
488):
489 """
490 [Docs](https://docs.litellm.ai/docs/pass_through/vllm)
491 """
492 from litellm.proxy.proxy_server import llm_router
494 request_body: Final = await get_request_body(request)
495 is_router_model: Final = is_passthrough_request_using_router_model(request_body, llm_router)
496 is_streaming_request: Final = is_passthrough_request_streaming(request_body)
497 if is_router_model and llm_router: 497 ↛ 498line 497 didn't jump to line 498 because the condition on line 497 was never true
498 result: Final = cast(
499 httpx.Response,
500 await llm_router.allm_passthrough_route(
501 model=request_body.get("model"),
502 method=request.method,
503 endpoint=endpoint,
504 request_query_params=request.query_params,
505 request_headers=_safe_get_request_headers(request),
506 stream=is_streaming_request,
507 content=None,
508 data=None,
509 files=None,
510 json=(request_body if is_json_content_type(request.headers.get("content-type", "")) else None),
511 params=None,
512 headers=None,
513 cookies=None,
514 litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict),
515 ),
516 )
518 if is_streaming_request:
519 return StreamingResponse(
520 content=result.aiter_bytes(),
521 status_code=result.status_code,
522 headers=HttpPassThroughEndpointHelpers.get_response_headers(
523 headers=result.headers,
524 custom_headers=None,
525 ),
526 )
528 content: Final = await result.aread()
529 return Response(
530 content=content,
531 status_code=result.status_code,
532 headers=HttpPassThroughEndpointHelpers.get_response_headers(
533 headers=result.headers,
534 custom_headers=None,
535 ),
536 )
538 return await llm_passthrough_factory_proxy_route(
539 endpoint=endpoint,
540 request=request,
541 fastapi_response=fastapi_response,
542 user_api_key_dict=user_api_key_dict,
543 custom_llm_provider="vllm",
544 )
547@router.api_route(
548 "/mistral/{endpoint:path}",
549 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
550 tags=["Mistral Pass-through", "pass-through"],
551)
552async def mistral_proxy_route(
553 endpoint: str,
554 request: Request,
555 fastapi_response: Response,
556 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
557):
558 """
559 [Docs](https://docs.litellm.ai/docs/pass_through/mistral)
560 """
561 base_target_url: Final = os.getenv("MISTRAL_API_BASE") or "https://api.mistral.ai"
562 encoded_endpoint = httpx.URL(endpoint).path
564 # Ensure endpoint starts with '/' for proper URL construction
565 if not encoded_endpoint.startswith("/"):
566 encoded_endpoint = "/" + encoded_endpoint
568 # Construct the full target URL using httpx, preserving any base path
569 # prefix that the operator configured on base_target_url.
570 base_url: Final = httpx.URL(base_target_url)
571 updated_url: Final = base_url.copy_with(
572 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
573 )
575 # Add or update query parameters
576 mistral_api_key: Final = passthrough_endpoint_router.get_credentials(
577 custom_llm_provider="mistral",
578 region_name=None,
579 )
581 ## check for streaming
582 is_streaming_request: Final = await is_streaming_request_fn(request)
584 ## CREATE PASS-THROUGH
585 endpoint_func: Final = create_pass_through_route(
586 endpoint=endpoint,
587 target=str(updated_url),
588 custom_headers={"Authorization": f"Bearer {mistral_api_key}"},
589 is_streaming_request=is_streaming_request,
590 ) # dynamically construct pass-through endpoint based on incoming path
591 received_value: Final = await endpoint_func(
592 request,
593 fastapi_response,
594 user_api_key_dict,
595 )
597 return received_value
600@router.api_route(
601 "/typesafe/{endpoint:path}",
602 methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list
603 tags=["TypeSafe AI Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list
604)
605async def typesafe_proxy_route(
606 endpoint: str,
607 request: Request,
608 fastapi_response: Response,
609 user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
610):
611 """[Docs](https://docs.litellm.ai/docs/pass_through/typesafe)"""
612 base_target_url: Final = get_secret_str("TYPESAFE_API_BASE") or "https://api.typesafe.ai"
613 encoded_endpoint: Final = httpx.URL(endpoint).path
614 normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}"
615 base_url: Final = httpx.URL(base_target_url)
616 updated_url: Final = base_url.copy_with(
617 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint),
618 )
619 typesafe_api_key: Final = passthrough_endpoint_router.get_credentials(
620 custom_llm_provider="typesafe",
621 region_name=None,
622 )
623 endpoint_func: Final = create_pass_through_route(
624 endpoint=endpoint,
625 target=str(updated_url),
626 custom_headers={ # mutable-ok: pass-through request headers require a mutable mapping
627 "Authorization": f"Bearer {typesafe_api_key}",
628 "Content-Type": "application/json",
629 },
630 custom_llm_provider="typesafe",
631 is_streaming_request=False,
632 )
633 return await endpoint_func(request, fastapi_response, user_api_key_dict)
636@router.api_route(
637 "/openrouter/{endpoint:path}",
638 methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list
639 tags=["OpenRouter Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list
640)
641async def openrouter_proxy_route(
642 endpoint: str,
643 request: Request,
644 fastapi_response: Response,
645 user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
646):
647 base_target_url: Final = get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1"
648 api_root: Final = base_target_url.removesuffix("/").removesuffix("/v1")
649 encoded_endpoint: Final = httpx.URL(endpoint).path
650 normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}"
651 base_url: Final = httpx.URL(api_root)
652 updated_url: Final = base_url.copy_with(
653 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint),
654 )
655 openrouter_api_key: Final = passthrough_endpoint_router.get_credentials(
656 custom_llm_provider="openrouter",
657 region_name=None,
658 )
659 endpoint_func: Final = create_pass_through_route(
660 endpoint=endpoint,
661 target=str(updated_url),
662 custom_headers={ # mutable-ok: pass-through request headers require a mutable mapping
663 "Authorization": f"Bearer {openrouter_api_key}",
664 "Content-Type": "application/json",
665 },
666 custom_llm_provider="openrouter",
667 is_streaming_request=False,
668 )
669 return await endpoint_func(request, fastapi_response, user_api_key_dict)
672@router.api_route(
673 "/milvus/{endpoint:path}",
674 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
675 tags=["Milvus Pass-through", "pass-through"],
676)
677async def milvus_proxy_route(
678 endpoint: str,
679 request: Request,
680 fastapi_response: Response,
681 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
682):
683 """
684 Enable using Milvus `/vectors` endpoint as a pass-through endpoint.
685 """
687 provider_config: Final = ProviderConfigManager.get_provider_vector_stores_config(provider=LlmProviders.MILVUS)
688 if not provider_config: 688 ↛ 689line 688 didn't jump to line 689 because the condition on line 688 was never true
689 raise HTTPException(
690 status_code=500,
691 detail="Unable to find Milvus vector store config.",
692 )
694 # check if managed vector store index is used
695 request_body: Final = await get_request_body(request)
697 # check collectionName
698 _raw_collection_name: Final = request_body.get("collectionName")
699 if _raw_collection_name is not None and not isinstance(_raw_collection_name, str): 699 ↛ 700line 699 didn't jump to line 700 because the condition on line 699 was never true
700 raise HTTPException(
701 status_code=400,
702 detail=f"collectionName must be a string. Got {type(_raw_collection_name).__name__}",
703 )
704 collection_name: str | None = _raw_collection_name # rebind-ok: locally scoped conversion
705 extra_headers = {} # mutable-ok: dict for extra headers; rebind-ok: reassigned later from credentials
706 base_target_url: str | None = None
707 if not collection_name: 707 ↛ 713line 707 didn't jump to line 713 because the condition on line 707 was always true
708 raise HTTPException(
709 status_code=400,
710 detail=f"Collection name is required. Got {request_body}",
711 )
713 if not litellm.vector_store_index_registry or not litellm.vector_store_registry:
714 raise HTTPException(
715 status_code=500,
716 detail="Unable to find Milvus vector store index registry or vector store registry.",
717 )
719 # check if vector store index
720 is_vector_store_index: Final = litellm.vector_store_index_registry.is_vector_store_index(
721 vector_store_index_name=collection_name
722 )
724 if not is_vector_store_index:
725 raise HTTPException(
726 status_code=400,
727 detail=f"Collection {collection_name} is not a litellm managed vector store index. Only litellm managed vector store indexes are supported.",
728 )
730 is_allowed_to_call_vector_store_endpoint(
731 index_name=collection_name,
732 provider=LlmProviders.MILVUS,
733 request=request,
734 user_api_key_dict=user_api_key_dict,
735 )
736 # get the vector store name from index registry
738 index_object: Final = (
739 (litellm.vector_store_index_registry.get_vector_store_index_by_name(vector_store_index_name=collection_name))
740 if litellm.vector_store_index_registry is not None
741 else None
742 )
743 if index_object is None:
744 raise Exception(f"Vector store index not found for {collection_name}")
746 vector_store_name: Final = index_object.litellm_params.vector_store_name
747 vector_store_index: Final = index_object.litellm_params.vector_store_index
749 request_body["collectionName"] = vector_store_index
751 # Update the request object with the modified collection name
752 _safe_set_request_parsed_body(request, request_body)
754 vector_store: Final = litellm.vector_store_registry.get_litellm_managed_vector_store_from_registry_by_name(
755 vector_store_name=vector_store_name
756 )
757 if vector_store is None:
758 raise Exception(f"Vector store not found for {vector_store_name}")
759 await assert_user_can_access_vector_store(
760 vector_store=vector_store,
761 user_api_key_dict=user_api_key_dict,
762 )
763 litellm_params = vector_store.get("litellm_params") or {}
764 auth_credentials: Final = provider_config.get_auth_credentials(litellm_params=litellm_params)
766 extra_headers = auth_credentials.get("headers") or {}
768 litellm_params = vector_store.get("litellm_params") or {}
770 base_target_url = provider_config.get_complete_url(
771 api_base=litellm_params.get("api_base"), litellm_params=litellm_params
772 )
774 if base_target_url is None:
775 raise Exception(f"api_base not found in vector store configuration for {vector_store_name}")
777 encoded_endpoint = httpx.URL(endpoint).path
779 # Ensure endpoint starts with '/' for proper URL construction
780 if not encoded_endpoint.startswith("/"):
781 encoded_endpoint = "/" + encoded_endpoint
783 # Construct the full target URL using httpx, preserving any base path
784 # prefix that the operator configured on base_target_url.
785 base_url: Final = httpx.URL(base_target_url)
786 updated_url: Final = base_url.copy_with(
787 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
788 )
789 ## CREATE PASS-THROUGH
790 endpoint_func: Final = create_pass_through_route(
791 endpoint=endpoint,
792 target=str(updated_url),
793 custom_headers=extra_headers,
794 ) # dynamically construct pass-through endpoint based on incoming path
795 received_value: Final = await endpoint_func(
796 request,
797 fastapi_response,
798 user_api_key_dict,
799 )
801 return received_value
804async def is_streaming_request_fn(request: Request) -> bool:
805 if request.method == "POST":
806 content_type: Final = request.headers.get("content-type", None)
807 if content_type and "multipart/form-data" in content_type: 807 ↛ 808line 807 didn't jump to line 808 because the condition on line 807 was never true
808 _request_body = await get_form_data(request)
809 else:
810 _request_body = await _read_request_body(request)
811 return is_passthrough_request_streaming(_request_body)
812 return False
815@router.api_route(
816 "/anthropic/{endpoint:path}",
817 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
818 tags=["Anthropic Pass-through", "pass-through"],
819)
820async def anthropic_proxy_route(
821 endpoint: str,
822 request: Request,
823 fastapi_response: Response,
824 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
825):
826 """
827 [Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)
828 """
829 base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
830 encoded_endpoint = httpx.URL(endpoint).path
832 # Ensure endpoint starts with '/' for proper URL construction
833 if not encoded_endpoint.startswith("/"): 833 ↛ 838line 833 didn't jump to line 838 because the condition on line 833 was always true
834 encoded_endpoint = "/" + encoded_endpoint
836 # Construct the full target URL using httpx, preserving any base path
837 # prefix that the operator configured on base_target_url.
838 base_url: Final = httpx.URL(base_target_url)
839 updated_url: Final = base_url.copy_with(
840 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
841 )
843 # Add or update query parameters
844 anthropic_api_key: Final = passthrough_endpoint_router.get_credentials(
845 custom_llm_provider="anthropic",
846 region_name=None,
847 )
849 ## check for streaming
850 is_streaming_request: Final = await is_streaming_request_fn(request)
852 ## CREATE PASS-THROUGH
853 auth_header: Final = AnthropicModelInfo.get_auth_header(anthropic_api_key or None)
854 endpoint_func: Final = create_pass_through_route(
855 endpoint=endpoint,
856 target=str(updated_url),
857 custom_headers=_upstream_headers_for_anthropic_route(request, user_api_key_dict, auth_header),
858 is_streaming_request=is_streaming_request,
859 ) # dynamically construct pass-through endpoint based on incoming path
860 received_value: Final = await endpoint_func(
861 request,
862 fastapi_response,
863 user_api_key_dict,
864 )
866 return received_value
869# Bedrock endpoint actions - consolidated list used for model extraction and streaming detection
870BEDROCK_ENDPOINT_ACTIONS: Final = {
871 "invoke",
872 "invoke-with-response-stream",
873 "converse",
874 "converse-stream",
875 "count_tokens",
876 "count-tokens",
877}
879BEDROCK_STREAMING_ACTIONS: Final = {"invoke-with-response-stream", "converse-stream"}
882def is_bedrock_count_tokens_endpoint(endpoint: str) -> bool:
883 return "count_tokens" in endpoint or "count-tokens" in endpoint
886def _extract_model_from_bedrock_endpoint(endpoint: str) -> str:
887 """
888 Extract model name from Bedrock endpoint path.
890 Handles model names with slashes (e.g., aws/anthropic/bedrock-claude-3-5-sonnet-v1)
891 by finding the action in the endpoint and extracting everything between "model" and the action.
893 Args:
894 endpoint: The endpoint path (e.g., "/model/aws/anthropic/model-name/invoke" or "v2/model/model-name/invoke")
896 Returns:
897 The extracted model name (e.g., "aws/anthropic/model-name" or "model-name")
899 Raises:
900 ValueError: If model cannot be extracted from endpoint
901 """
902 try:
903 endpoint_parts: Final = endpoint.split("/")
905 if "application-inference-profile" in endpoint: 905 ↛ 907line 905 didn't jump to line 907 because the condition on line 905 was never true
906 # Format: model/application-inference-profile/{profile-id}/{action}
907 return "/".join(endpoint_parts[1:3])
909 # Format: model/{modelId}/{action} or v2/model/{modelId}/{action}
910 # Find the index of "model" in the endpoint parts
911 model_index = None
912 for idx, part in enumerate(endpoint_parts):
913 if part == "model": 913 ↛ 914line 913 didn't jump to line 914 because the condition on line 913 was never true
914 model_index = idx
915 break
917 # If "model" keyword not found, try to extract model from the endpoint
918 # by finding the action and taking everything before it
919 if model_index is None: 919 ↛ 938line 919 didn't jump to line 938 because the condition on line 919 was always true
920 # Find the index of the action in the endpoint parts
921 action_index = None
922 for idx, part in enumerate(endpoint_parts):
923 if part in BEDROCK_ENDPOINT_ACTIONS: 923 ↛ 924line 923 didn't jump to line 924 because the condition on line 923 was never true
924 action_index = idx
925 break
927 if action_index is not None and action_index > 1: 927 ↛ 929line 927 didn't jump to line 929 because the condition on line 927 was never true
928 # Join all parts before the action (excluding empty strings)
929 model_parts = [p for p in endpoint_parts[1:action_index] if p]
930 if model_parts:
931 return "/".join(model_parts)
933 raise ValueError(
934 f"'model' keyword not found and unable to extract model from endpoint. Expected format: /model/{{modelId}}/{{action}}. Got: {endpoint}"
935 )
937 # Find the index of the action in the endpoint parts
938 action_index = None
939 for idx, part in enumerate(endpoint_parts):
940 if part in BEDROCK_ENDPOINT_ACTIONS:
941 action_index = idx
942 break
944 if action_index is not None and action_index > model_index + 1:
945 # Join all parts between "model" and the action (excluding "model" itself)
946 return "/".join(endpoint_parts[model_index + 1 : action_index])
948 # Fallback to taking everything after "model" if no action found
949 model_parts = [p for p in endpoint_parts[model_index + 1 :] if p]
950 if model_parts:
951 return "/".join(model_parts)
953 raise ValueError(
954 f"No model ID found after 'model' keyword. Expected format: /model/{{modelId}}/{{action}}. Got: {endpoint}"
955 )
957 except ValueError:
958 # Re-raise ValueError as-is
959 raise
960 except Exception as e:
961 raise ValueError(
962 f"Model missing from endpoint. Expected format: /model/{{modelId}}/{{action}}. Got: {endpoint}"
963 ) from e
966async def handle_bedrock_passthrough_router_model(
967 model: str,
968 endpoint: str,
969 request: Request,
970 request_body: dict,
971 llm_router: litellm.Router,
972 user_api_key_dict: UserAPIKeyAuth,
973 proxy_logging_obj,
974 general_settings: dict,
975 proxy_config,
976 select_data_generator,
977 user_model: str | None,
978 user_temperature: float | None,
979 user_request_timeout: float | None,
980 user_max_tokens: int | None,
981 user_api_base: str | None,
982 version: str | None,
983) -> Response | StreamingResponse:
984 """
985 Handle Bedrock passthrough for router models (models defined in config.yaml).
987 Uses the same common processing path as non-router models to ensure
988 metadata and hooks are properly initialized.
990 Args:
991 model: The router model name (e.g., "aws/anthropic/bedrock-claude-3-5-sonnet-v1")
992 endpoint: The Bedrock endpoint path (e.g., "/model/{modelId}/invoke")
993 request: The FastAPI request object
994 request_body: The parsed request body
995 llm_router: The LiteLLM router instance
996 user_api_key_dict: The user API key authentication dictionary
997 (additional args for common processing)
999 Returns:
1000 Response or StreamingResponse depending on endpoint type
1001 """
1002 from fastapi import Response as FastAPIResponse
1004 from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
1006 # Detect streaming based on endpoint
1007 is_streaming: Final = any(action in endpoint for action in BEDROCK_STREAMING_ACTIONS)
1009 verbose_proxy_logger.debug(
1010 "Bedrock router passthrough: model='%s', endpoint='%s', streaming=%s", model, endpoint, is_streaming
1011 )
1013 # Use the common processing path (same as non-router models)
1014 # This ensures all metadata, hooks, and logging are properly initialized
1015 data: Final[dict[str, object]] = {}
1016 base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
1018 data["model"] = model
1019 data["method"] = request.method
1020 data["endpoint"] = endpoint
1021 data["data"] = request_body
1022 data["custom_llm_provider"] = "bedrock"
1024 # Use the common passthrough processing to handle metadata and hooks
1025 # This also handles all response formatting (streaming/non-streaming) and exceptions
1026 try:
1027 result: Final = await base_llm_response_processor.base_passthrough_process_llm_request(
1028 request=request,
1029 fastapi_response=FastAPIResponse(),
1030 user_api_key_dict=user_api_key_dict,
1031 proxy_logging_obj=proxy_logging_obj,
1032 llm_router=llm_router,
1033 general_settings=general_settings,
1034 proxy_config=proxy_config,
1035 select_data_generator=select_data_generator,
1036 model=model,
1037 user_model=user_model,
1038 user_temperature=user_temperature,
1039 user_request_timeout=user_request_timeout,
1040 user_max_tokens=user_max_tokens,
1041 user_api_base=user_api_base,
1042 version=version,
1043 )
1044 return result
1045 except Exception as e:
1046 # Use common exception handling
1047 raise await base_llm_response_processor._handle_llm_api_exception(
1048 e=e,
1049 user_api_key_dict=user_api_key_dict,
1050 proxy_logging_obj=proxy_logging_obj,
1051 )
1054async def handle_bedrock_count_tokens(
1055 endpoint: str,
1056 request: Request,
1057 fastapi_response: Response,
1058 user_api_key_dict: UserAPIKeyAuth,
1059 request_body: dict[str, object],
1060) -> dict[str, object]:
1061 """
1062 Handle AWS Bedrock CountTokens API requests.
1064 This function processes count_tokens endpoints like:
1065 - /v1/messages/count_tokens
1066 - /v1/messages/count-tokens
1067 """
1068 from litellm.llms.bedrock.common_utils import BedrockError
1069 from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
1070 from litellm.proxy.proxy_server import llm_router
1072 try:
1073 # Initialize the handler
1074 handler: Final = BedrockCountTokensHandler()
1076 # Extract model from request body
1077 model: Final = _optional_str(request_body.get("model"))
1078 if not model:
1079 raise HTTPException(status_code=400, detail={"error": "Model is required in request body"})
1081 # Get model parameters from router
1082 litellm_params: Final = {"user_api_key_dict": user_api_key_dict}
1083 resolved_model = model # Default fallback
1085 if llm_router:
1086 deployments: Final = llm_router.get_model_list(model_name=model)
1087 if deployments and len(deployments) > 0:
1088 # Get the first matching deployment
1089 deployment: Final = deployments[0]
1090 model_litellm_params: Final = deployment.get("litellm_params", {})
1092 # Get the resolved model ID from the configuration
1093 if "model" in model_litellm_params:
1094 resolved_model = model_litellm_params["model"]
1096 # Copy all litellm_params - BaseAWSLLM will handle AWS credential discovery
1097 for key, value in model_litellm_params.items():
1098 if key != "user_api_key_dict": # Don't overwrite user_api_key_dict
1099 litellm_params[key] = value
1101 verbose_proxy_logger.debug("Count tokens litellm_params: %s", litellm_params)
1102 verbose_proxy_logger.debug("Resolved model: %s", resolved_model)
1104 # Handle the count tokens request
1105 result: Final = await handler.handle_count_tokens_request(
1106 request_data=request_body,
1107 litellm_params=litellm_params,
1108 resolved_model=resolved_model,
1109 )
1111 return result
1113 except BedrockError as e:
1114 # Convert BedrockError to HTTPException for FastAPI
1115 verbose_proxy_logger.error("BedrockError in handle_bedrock_count_tokens: %s", e)
1116 from litellm.litellm_core_utils.llm_response_utils.get_headers import get_response_headers
1118 provider_headers: Final = getattr(getattr(e, "response", None), "headers", None)
1119 raise HTTPException(
1120 status_code=e.status_code,
1121 detail={"error": e.message},
1122 headers=get_response_headers(provider_headers) if provider_headers else None,
1123 )
1124 except HTTPException:
1125 # Re-raise HTTP exceptions as-is
1126 raise
1127 except Exception as e:
1128 verbose_proxy_logger.error("Error in handle_bedrock_count_tokens: %s", e)
1129 raise HTTPException(status_code=500, detail={"error": f"CountTokens processing error: {e}"})
1132async def bedrock_llm_proxy_route(
1133 endpoint: str,
1134 request: Request,
1135 fastapi_response: Response,
1136 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
1137):
1138 """
1139 Handles Bedrock LLM API calls.
1141 Supports both direct Bedrock models and router models from config.yaml.
1143 Endpoints:
1144 - /model/{modelId}/invoke
1145 - /model/{modelId}/invoke-with-response-stream
1146 - /model/{modelId}/converse
1147 - /model/{modelId}/converse-stream
1148 - /model/application-inference-profile/{profileId}/{action}
1149 """
1150 from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
1151 from litellm.proxy.proxy_server import (
1152 general_settings,
1153 llm_router,
1154 proxy_config,
1155 proxy_logging_obj,
1156 select_data_generator,
1157 user_api_base,
1158 user_max_tokens,
1159 user_model,
1160 user_request_timeout,
1161 user_temperature,
1162 version,
1163 )
1165 request_body: Final = await _read_request_body(request=request)
1167 if is_bedrock_count_tokens_endpoint(endpoint): 1167 ↛ 1168line 1167 didn't jump to line 1168 because the condition on line 1167 was never true
1168 return await handle_bedrock_count_tokens(
1169 endpoint=endpoint,
1170 request=request,
1171 fastapi_response=fastapi_response,
1172 user_api_key_dict=user_api_key_dict,
1173 request_body=request_body,
1174 )
1176 # Extract model from endpoint path using helper
1177 try:
1178 model: Final = _extract_model_from_bedrock_endpoint(endpoint=endpoint)
1179 except ValueError as e:
1180 raise HTTPException(
1181 status_code=400,
1182 detail={"error": str(e)},
1183 )
1185 # Check if this is a router model (from config.yaml)
1186 is_router_model = is_passthrough_request_using_router_model(request_body={"model": model}, llm_router=llm_router)
1188 # If router model, use dedicated router passthrough handler
1189 # This uses the same common processing path as non-router models
1190 if is_router_model and llm_router:
1191 return await handle_bedrock_passthrough_router_model(
1192 model=model,
1193 endpoint=endpoint,
1194 request=request,
1195 request_body=request_body,
1196 llm_router=llm_router,
1197 user_api_key_dict=user_api_key_dict,
1198 proxy_logging_obj=proxy_logging_obj,
1199 general_settings=general_settings,
1200 proxy_config=proxy_config,
1201 select_data_generator=select_data_generator,
1202 user_model=user_model,
1203 user_temperature=user_temperature,
1204 user_request_timeout=user_request_timeout,
1205 user_max_tokens=user_max_tokens,
1206 user_api_base=user_api_base,
1207 version=version,
1208 )
1210 # Fall back to existing implementation for direct Bedrock models
1211 verbose_proxy_logger.debug(
1212 "Bedrock passthrough: Using direct Bedrock model '%s' for endpoint '%s'", model, endpoint
1213 )
1215 data: Final[dict[str, object]] = {}
1216 base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
1218 data["method"] = request.method
1219 data["endpoint"] = endpoint
1220 data["data"] = request_body
1221 data["custom_llm_provider"] = "bedrock"
1223 try:
1224 result: Final = await base_llm_response_processor.base_passthrough_process_llm_request(
1225 request=request,
1226 fastapi_response=fastapi_response,
1227 user_api_key_dict=user_api_key_dict,
1228 proxy_logging_obj=proxy_logging_obj,
1229 llm_router=llm_router,
1230 general_settings=general_settings,
1231 proxy_config=proxy_config,
1232 select_data_generator=select_data_generator,
1233 model=model,
1234 user_model=user_model,
1235 user_temperature=user_temperature,
1236 user_request_timeout=user_request_timeout,
1237 user_max_tokens=user_max_tokens,
1238 user_api_base=user_api_base,
1239 version=version,
1240 )
1242 return result
1243 except Exception as e:
1244 raise await base_llm_response_processor._handle_llm_api_exception(
1245 e=e,
1246 user_api_key_dict=user_api_key_dict,
1247 proxy_logging_obj=proxy_logging_obj,
1248 )
1251@router.api_route(
1252 "/bedrock/{endpoint:path}",
1253 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
1254 tags=["Bedrock Pass-through", "pass-through"],
1255)
1256async def bedrock_proxy_route(
1257 endpoint: str,
1258 request: Request,
1259 fastapi_response: Response,
1260 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
1261):
1262 """
1263 This is the v1 passthrough for Bedrock.
1264 V2 is handled by the `/bedrock/v2` endpoint.
1265 [Docs](https://docs.litellm.ai/docs/pass_through/bedrock)
1266 """
1267 create_request_copy(request)
1269 aws_region_name: Final = get_secret_str(secret_name="AWS_REGION_NAME")
1270 if not _is_bedrock_agent_runtime_route(endpoint=endpoint): 1270 ↛ 1278line 1270 didn't jump to line 1278 because the condition on line 1270 was always true
1271 return await bedrock_llm_proxy_route(
1272 endpoint=endpoint,
1273 request=request,
1274 fastapi_response=fastapi_response,
1275 user_api_key_dict=user_api_key_dict,
1276 )
1278 if _is_bedrock_agent_runtime_passthrough_disabled():
1279 raise HTTPException(
1280 status_code=403,
1281 detail="bedrock-agent-runtime pass-through is disabled on this proxy.",
1282 )
1284 base_target_url: Final = f"https://bedrock-agent-runtime.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}"
1285 encoded_endpoint = httpx.URL(endpoint).path
1287 # Ensure endpoint starts with '/' for proper URL construction
1288 if not encoded_endpoint.startswith("/"):
1289 encoded_endpoint = "/" + encoded_endpoint
1291 # Construct the full target URL using httpx, preserving any base path
1292 # prefix that the operator configured on base_target_url.
1293 base_url: Final = httpx.URL(base_target_url)
1294 updated_url: Final = base_url.copy_with(
1295 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
1296 )
1298 # Add or update query parameters
1299 from litellm.llms.bedrock.base_aws_llm import run_aws_signing, sign_aws_json_post
1300 from litellm.llms.bedrock.chat import BedrockConverseLLM
1302 bedrock_llm: Final = BedrockConverseLLM()
1303 # Assuming the body contains JSON data, parse it
1304 try:
1305 data: Final = await _json_request_body(request)
1306 except Exception as e:
1307 raise HTTPException(status_code=400, detail={"error": e})
1308 prepped: Final = await run_aws_signing(
1309 sign_aws_json_post,
1310 get_credentials=bedrock_llm.get_credentials,
1311 service_name="bedrock",
1312 aws_region_name=aws_region_name,
1313 url=str(updated_url),
1314 body=json.dumps(data),
1315 headers=MappingProxyType({"Content-Type": "application/json"}),
1316 )
1318 ## check for streaming
1319 is_streaming_request = False
1320 if "stream" in str(updated_url):
1321 is_streaming_request = True
1323 ## CREATE PASS-THROUGH
1324 endpoint_func: Final = create_pass_through_route(
1325 endpoint=endpoint,
1326 target=str(prepped.url),
1327 custom_headers=_upstream_headers_for_bedrock_agent_runtime_route(request, user_api_key_dict, prepped.headers),
1328 is_streaming_request=is_streaming_request,
1329 ) # dynamically construct pass-through endpoint based on incoming path
1330 setattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, data)
1331 # SigV4 signs an exact payload; pass-through must send prepped.body, not json.dumps
1332 # of a dict that hooks may mutate (logging_obj, metadata, etc.).
1333 setattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, prepped.body)
1334 received_value: Final = await endpoint_func(
1335 request,
1336 fastapi_response,
1337 user_api_key_dict,
1338 )
1340 return received_value
1343COMPREHEND_MEDICAL_TARGET_PREFIX: Final = "ComprehendMedical_20181030"
1346def _proxy_general_settings() -> Mapping[str, object]:
1347 from litellm.proxy.proxy_server import general_settings
1349 return general_settings
1352def _resolve_aws_passthrough_region() -> str | None:
1353 region_candidates: Final = (
1354 get_secret_str(secret_name="AWS_REGION_NAME"),
1355 get_secret_str(secret_name="AWS_REGION"),
1356 get_secret_str(secret_name="AWS_DEFAULT_REGION"),
1357 )
1358 return next((region for region in region_candidates if region), None)
1361@router.post(
1362 "/comprehendmedical/{operation}",
1363 tags=["AWS Comprehend Medical Pass-through", "pass-through"], # mutable-ok: fastapi route tags must be a list
1364)
1365async def comprehend_medical_proxy_route(
1366 operation: str,
1367 request: Request,
1368 fastapi_response: Response,
1369 user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
1370):
1371 """
1372 Pass-through for Amazon Comprehend Medical, e.g. `POST /comprehendmedical/DetectEntitiesV2`.
1374 The request body is forwarded as-is to the AWS JSON 1.1 API and signed with SigV4
1375 using the proxy's AWS credentials.
1377 [Docs](https://docs.litellm.ai/docs/pass_through/comprehend_medical)
1378 """
1379 from .llm_provider_handlers.comprehend_medical_passthrough_logging_handler import (
1380 COMPREHEND_MEDICAL_SUPPORTED_OPERATIONS,
1381 )
1383 if operation not in COMPREHEND_MEDICAL_SUPPORTED_OPERATIONS: 1383 ↛ 1392line 1383 didn't jump to line 1392 because the condition on line 1383 was always true
1384 raise HTTPException(
1385 status_code=400,
1386 detail=(
1387 f"Unsupported Comprehend Medical operation: {operation}. "
1388 f"Supported operations: {', '.join(sorted(COMPREHEND_MEDICAL_SUPPORTED_OPERATIONS))}"
1389 ),
1390 )
1392 aws_region_name: Final = _resolve_aws_passthrough_region()
1393 if aws_region_name is None:
1394 raise HTTPException(
1395 status_code=400,
1396 detail="AWS region not found. Set AWS_REGION_NAME in the proxy environment.",
1397 )
1399 try:
1400 data: Final = await _json_request_body(request)
1401 except Exception as e:
1402 raise HTTPException(status_code=400, detail=str(e))
1404 if not isinstance(data, dict):
1405 raise HTTPException(status_code=400, detail="Request body must be a JSON object")
1406 if "stream" in data:
1407 raise HTTPException(status_code=400, detail="'stream' is not a Comprehend Medical request member")
1409 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing, sign_aws_json_post
1411 target_url: Final = f"https://comprehendmedical.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}/"
1412 prepped: Final = await run_aws_signing(
1413 sign_aws_json_post,
1414 get_credentials=partial(BaseAWSLLM().get_credentials, aws_region_name=aws_region_name),
1415 service_name="comprehendmedical",
1416 aws_region_name=aws_region_name,
1417 url=target_url,
1418 body=json.dumps(data),
1419 headers=MappingProxyType(
1420 {
1421 "Content-Type": "application/x-amz-json-1.1",
1422 "X-Amz-Target": f"{COMPREHEND_MEDICAL_TARGET_PREFIX}.{operation}",
1423 }
1424 ),
1425 )
1427 endpoint_func: Final = create_pass_through_route(
1428 endpoint=operation,
1429 target=str(prepped.url),
1430 custom_headers=prepped.headers,
1431 custom_llm_provider="comprehendmedical",
1432 )
1433 setattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, data)
1434 setattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, prepped.body)
1435 return await endpoint_func(request, fastapi_response, user_api_key_dict)
1438@router.post(
1439 "/comprehendmedical",
1440 tags=["AWS Comprehend Medical Pass-through", "pass-through"], # mutable-ok: fastapi route tags must be a list
1441)
1442async def comprehend_medical_sdk_proxy_route(
1443 request: Request,
1444 fastapi_response: Response,
1445 user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
1446):
1447 """
1448 AWS-SDK-shaped pass-through for Amazon Comprehend Medical: point the SDK's
1449 `endpoint_url` at `/comprehendmedical` and the operation is read from the
1450 `X-Amz-Target` header, per the AWS JSON 1.1 protocol.
1452 [Docs](https://docs.litellm.ai/docs/pass_through/comprehend_medical)
1453 """
1454 target_header: Final = request.headers.get("x-amz-target", "")
1455 target_prefix, _, operation = target_header.partition(".")
1456 if target_prefix != COMPREHEND_MEDICAL_TARGET_PREFIX or not operation: 1456 ↛ 1461line 1456 didn't jump to line 1461 because the condition on line 1456 was always true
1457 raise HTTPException(
1458 status_code=400,
1459 detail=f"Expected an X-Amz-Target header of the form {COMPREHEND_MEDICAL_TARGET_PREFIX}.<Operation>",
1460 )
1461 return await comprehend_medical_proxy_route(
1462 operation=operation,
1463 request=request,
1464 fastapi_response=fastapi_response,
1465 user_api_key_dict=user_api_key_dict,
1466 )
1469AZURE_SPEECH_FORWARDED_REQUEST_HEADERS: Final = ("content-type", "accept")
1470AZURE_SPEECH_ENDPOINT_FAMILY_DOMAINS: Final = MappingProxyType(
1471 {
1472 AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX: AZURE_SPEECH_STT_DOMAIN,
1473 AZURE_SPEECH_BATCH_PATH_PREFIX: AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN,
1474 }
1475)
1478def resolve_azure_speech_base_url(endpoint_path: str, api_base: str | None, region: str | None) -> httpx.URL | None:
1479 """
1480 Azure AI Speech serves the two REST families from different regional hosts: short-audio
1481 recognition under ``{region}.stt.speech.microsoft.com`` and batch transcription under
1482 ``{region}.api.cognitive.microsoft.com``. An operator-configured ``api_base`` (custom
1483 domain or private endpoint) serves both and wins over the region. Returns ``None`` when
1484 the path is outside both families so the operator key is never sent for an unknown API.
1485 """
1486 domain: Final = next(
1487 (
1488 family_domain
1489 for family_prefix, family_domain in AZURE_SPEECH_ENDPOINT_FAMILY_DOMAINS.items()
1490 if endpoint_path.startswith(family_prefix)
1491 ),
1492 None,
1493 )
1494 if domain is None: 1494 ↛ 1496line 1494 didn't jump to line 1496 because the condition on line 1494 was always true
1495 return None
1496 if api_base:
1497 return httpx.URL(api_base)
1498 if not region:
1499 return None
1500 return httpx.URL(f"https://{region}.{domain}")
1503def azure_speech_path_manages_shared_resources(endpoint_path: str) -> bool:
1504 return (
1505 endpoint_path.startswith(AZURE_SPEECH_BATCH_PATH_PREFIX)
1506 and endpoint_path != AZURE_SPEECH_FAST_TRANSCRIPTION_PATH
1507 )
1510def canonical_azure_speech_endpoint_path(endpoint: str) -> str:
1511 """
1512 The path Azure will actually serve, with ``.`` and ``..`` segments resolved, so the
1513 endpoint family and the admin guard are decided on the same path the upstream request uses.
1514 """
1515 raw_path: Final = httpx.URL(endpoint).path
1516 resolved_path: Final = posixpath.normpath(f"/{raw_path.lstrip('/')}")
1517 if raw_path.endswith("/") and resolved_path != "/": 1517 ↛ 1518line 1517 didn't jump to line 1518 because the condition on line 1517 was never true
1518 return f"{resolved_path}/"
1519 return resolved_path
1522@router.api_route(
1523 f"{AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX}/{{endpoint:path}}",
1524 methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: fastapi route methods must be a list
1525 tags=["Azure AI Speech Pass-through", "pass-through"], # mutable-ok: fastapi route tags must be a list
1526)
1527async def azure_speech_proxy_route(
1528 endpoint: str,
1529 request: Request,
1530 fastapi_response: Response,
1531 user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
1532):
1533 """
1534 Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.
1535 `POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`
1536 with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.
1538 The body is forwarded byte for byte and the proxy injects its own
1539 `Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key
1540 and is never forwarded.
1542 [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)
1543 """
1544 normalized_endpoint_path: Final = canonical_azure_speech_endpoint_path(endpoint)
1545 base_url: Final = resolve_azure_speech_base_url(
1546 endpoint_path=normalized_endpoint_path,
1547 api_base=get_secret_str(secret_name="AZURE_SPEECH_API_BASE"),
1548 region=get_secret_str(secret_name="AZURE_SPEECH_REGION"),
1549 )
1550 if base_url is None: 1550 ↛ 1559line 1550 didn't jump to line 1559 because the condition on line 1550 was always true
1551 raise HTTPException(
1552 status_code=400,
1553 detail=(
1554 f"Unsupported Azure Speech path: {normalized_endpoint_path}. Supported prefixes are "
1555 f"{AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX} and {AZURE_SPEECH_BATCH_PATH_PREFIX}; set "
1556 "AZURE_SPEECH_REGION or AZURE_SPEECH_API_BASE in the proxy environment."
1557 ),
1558 )
1559 if azure_speech_path_manages_shared_resources(normalized_endpoint_path) and not is_proxy_admin(user_api_key_dict):
1560 raise HTTPException(
1561 status_code=403,
1562 detail=(
1563 f"{request.method} {normalized_endpoint_path} manages batch transcription resources that belong to "
1564 "the proxy's Azure Speech subscription and whose cost is unknown at request time, so it is limited "
1565 f"to proxy admin keys. Use {AZURE_SPEECH_FAST_TRANSCRIPTION_PATH} for transcription that is priced "
1566 "per request."
1567 ),
1568 )
1569 azure_speech_api_key: Final = passthrough_endpoint_router.get_credentials(
1570 custom_llm_provider=AZURE_SPEECH_CUSTOM_LLM_PROVIDER,
1571 region_name=None,
1572 )
1573 if azure_speech_api_key is None:
1574 raise HTTPException(
1575 status_code=400,
1576 detail="Azure Speech credentials not found. Set AZURE_SPEECH_API_KEY in the proxy environment.",
1577 )
1579 target_url: Final = base_url.copy_with(
1580 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint_path)
1581 )
1582 request_headers: Final = _safe_get_request_headers(request)
1583 upstream_headers: Final = MappingProxyType(
1584 {
1585 header_name: header_value
1586 for header_name, header_value in (
1587 *(
1588 (header_name, request_headers[header_name])
1589 for header_name in AZURE_SPEECH_FORWARDED_REQUEST_HEADERS
1590 if header_name in request_headers
1591 ),
1592 (AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER, azure_speech_api_key),
1593 )
1594 }
1595 )
1596 raw_body: Final = await request.body()
1598 endpoint_func: Final = create_pass_through_route(
1599 endpoint=endpoint,
1600 target=str(target_url),
1601 custom_headers=upstream_headers,
1602 custom_llm_provider=AZURE_SPEECH_CUSTOM_LLM_PROVIDER,
1603 )
1604 setattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, raw_body)
1605 return await endpoint_func(request, fastapi_response, user_api_key_dict)
1608@router.post(
1609 "/transcribe/{operation}",
1610 tags=["Amazon Transcribe Pass-through", "pass-through"], # mutable-ok: fastapi route tags must be a list
1611)
1612async def transcribe_proxy_route(
1613 operation: str,
1614 request: Request,
1615 fastapi_response: Response,
1616 user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
1617 general_settings: Annotated[Mapping[str, object], Depends(_proxy_general_settings)],
1618):
1619 """
1620 Pass-through for the Amazon Transcribe API, e.g. `POST /transcribe/StartTranscriptionJob`.
1622 The request body is forwarded to the AWS JSON 1.1 API and signed with SigV4 using the
1623 proxy's AWS credentials. Standard jobs are tagged with the calling key's owner so that
1624 only that owner (or a proxy admin) can read or delete them, and keys other than proxy
1625 admins may only read media from and write transcripts to the S3 buckets listed in
1626 `general_settings.transcribe_media_buckets`; account-wide operations
1627 such as ListTranscriptionJobs are limited to proxy admins. Streaming transcription
1628 (`transcribestreaming`) uses a separate HTTP/2 event-stream protocol and is not served
1629 by this route.
1631 [Docs](https://docs.litellm.ai/docs/pass_through/transcribe)
1632 """
1633 from .llm_provider_handlers.transcribe_passthrough_logging_handler import (
1634 TRANSCRIBE_CUSTOM_LLM_PROVIDER,
1635 TRANSCRIBE_OWNED_JOB_OPERATIONS,
1636 TRANSCRIBE_PRICED_OPERATION,
1637 TRANSCRIBE_TARGET_PREFIX,
1638 TranscribeRefusal,
1639 transcribe_admin_only_refusal,
1640 transcribe_cost_per_second,
1641 transcribe_job_access_refusal,
1642 transcribe_job_lookup,
1643 transcribe_media_buckets,
1644 transcribe_owned_start_request,
1645 transcribe_storage_refusal,
1646 transcribe_supported_operations,
1647 transcribe_unpriceable_request_reason,
1648 )
1650 if operation not in transcribe_supported_operations(): 1650 ↛ 1659line 1650 didn't jump to line 1659 because the condition on line 1650 was always true
1651 raise HTTPException(
1652 status_code=400,
1653 detail=(
1654 f"Unsupported Amazon Transcribe operation: {operation}. "
1655 f"Supported operations: {', '.join(sorted(transcribe_supported_operations()))}"
1656 ),
1657 )
1659 aws_region_name: Final = _resolve_aws_passthrough_region()
1660 if aws_region_name is None:
1661 raise HTTPException(
1662 status_code=400,
1663 detail="AWS region not found. Set AWS_REGION_NAME in the proxy environment.",
1664 )
1666 try:
1667 data: Final = await _json_request_body(request)
1668 except ValueError as e:
1669 raise HTTPException(status_code=400, detail=f"Request body must be valid JSON: {e}")
1671 if not isinstance(data, dict):
1672 raise HTTPException(status_code=400, detail="Request body must be a JSON object")
1673 if "stream" in data:
1674 raise HTTPException(status_code=400, detail="'stream' is not an Amazon Transcribe request member")
1675 unpriceable_reason: Final = transcribe_unpriceable_request_reason(operation, data, transcribe_cost_per_second())
1676 if unpriceable_reason is not None:
1677 raise HTTPException(status_code=400, detail=unpriceable_reason)
1678 admin_only_refusal: Final = transcribe_admin_only_refusal(operation, user_api_key_dict)
1679 if admin_only_refusal is not None:
1680 raise HTTPException(status_code=admin_only_refusal.status_code, detail=admin_only_refusal.detail)
1681 storage_refusal: Final = (
1682 transcribe_storage_refusal(data, transcribe_media_buckets(general_settings), user_api_key_dict)
1683 if operation == TRANSCRIBE_PRICED_OPERATION
1684 else None
1685 )
1686 if storage_refusal is not None:
1687 raise HTTPException(status_code=storage_refusal.status_code, detail=storage_refusal.detail)
1688 request_body: Final = (
1689 transcribe_owned_start_request(data, user_api_key_dict) if operation == TRANSCRIBE_PRICED_OPERATION else data
1690 )
1691 if isinstance(request_body, TranscribeRefusal):
1692 raise HTTPException(status_code=request_body.status_code, detail=request_body.detail)
1693 access_refusal: Final = (
1694 await transcribe_job_access_refusal(
1695 data.get("TranscriptionJobName"), user_api_key_dict, transcribe_job_lookup(aws_region_name)
1696 )
1697 if operation in TRANSCRIBE_OWNED_JOB_OPERATIONS
1698 else None
1699 )
1700 if access_refusal is not None:
1701 raise HTTPException(status_code=access_refusal.status_code, detail=access_refusal.detail)
1703 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing, sign_aws_json_post
1705 target_url: Final = f"https://transcribe.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}/"
1706 prepped: Final = await run_aws_signing(
1707 sign_aws_json_post,
1708 get_credentials=partial(BaseAWSLLM().get_credentials, aws_region_name=aws_region_name),
1709 service_name="transcribe",
1710 aws_region_name=aws_region_name,
1711 url=target_url,
1712 body=json.dumps(request_body),
1713 headers=MappingProxyType(
1714 {
1715 "Content-Type": "application/x-amz-json-1.1",
1716 "X-Amz-Target": f"{TRANSCRIBE_TARGET_PREFIX}.{operation}",
1717 }
1718 ),
1719 )
1721 endpoint_func: Final = create_pass_through_route(
1722 endpoint=operation,
1723 target=str(prepped.url),
1724 custom_headers=prepped.headers,
1725 custom_llm_provider=TRANSCRIBE_CUSTOM_LLM_PROVIDER,
1726 )
1727 setattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, request_body)
1728 setattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, prepped.body)
1729 return await endpoint_func(request, fastapi_response, user_api_key_dict)
1732@router.post(
1733 "/transcribe",
1734 tags=["Amazon Transcribe Pass-through", "pass-through"], # mutable-ok: fastapi route tags must be a list
1735)
1736async def transcribe_sdk_proxy_route(
1737 request: Request,
1738 fastapi_response: Response,
1739 user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
1740 general_settings: Annotated[Mapping[str, object], Depends(_proxy_general_settings)],
1741):
1742 """
1743 AWS-SDK-shaped pass-through for Amazon Transcribe: point the SDK's `endpoint_url`
1744 at `/transcribe` and the operation is read from the `X-Amz-Target` header, per the
1745 AWS JSON 1.1 protocol.
1747 [Docs](https://docs.litellm.ai/docs/pass_through/transcribe)
1748 """
1749 from .llm_provider_handlers.transcribe_passthrough_logging_handler import (
1750 TRANSCRIBE_TARGET_PREFIX,
1751 )
1753 target_header: Final = request.headers.get("x-amz-target", "")
1754 target_prefix, _, operation = target_header.partition(".")
1755 if target_prefix != TRANSCRIBE_TARGET_PREFIX or not operation: 1755 ↛ 1760line 1755 didn't jump to line 1760 because the condition on line 1755 was always true
1756 raise HTTPException(
1757 status_code=400,
1758 detail=f"Expected an X-Amz-Target header of the form {TRANSCRIBE_TARGET_PREFIX}.<Operation>",
1759 )
1760 return await transcribe_proxy_route(
1761 operation=operation,
1762 request=request,
1763 fastapi_response=fastapi_response,
1764 user_api_key_dict=user_api_key_dict,
1765 general_settings=general_settings,
1766 )
1769def _resolve_vertex_model_from_router(
1770 model_id: str,
1771 llm_router: litellm.Router | None,
1772 encoded_endpoint: str,
1773 endpoint: str,
1774 vertex_project: str | None,
1775 vertex_location: str | None,
1776) -> tuple[str, str, str | None, str | None, Mapping[str, object] | None]:
1777 """
1778 Resolve Vertex AI model configuration from router.
1780 Args:
1781 model_id: The model ID extracted from the URL (e.g., "gcp/google/gemini-2.5-flash")
1782 llm_router: The LiteLLM router instance
1783 encoded_endpoint: The encoded endpoint path
1784 endpoint: The original endpoint path
1785 vertex_project: Current vertex project (may be from URL)
1786 vertex_location: Current vertex location (may be from URL)
1788 Returns:
1789 tuple of (encoded_endpoint, endpoint, vertex_project, vertex_location, deployment_model_info)
1790 with resolved values from router config; deployment_model_info is the resolved
1791 deployment's `model_info`, or None when no deployment matched
1792 """
1793 if not llm_router:
1794 return encoded_endpoint, endpoint, vertex_project, vertex_location, None
1796 try:
1797 deployment: Final = llm_router.get_available_deployment_for_pass_through(model=model_id)
1798 if not deployment:
1799 return encoded_endpoint, endpoint, vertex_project, vertex_location, None
1801 litellm_params: Final = deployment.get("litellm_params", {})
1802 model_info: Final = deployment.get("model_info")
1803 deployment_model_info: Final = model_info if isinstance(model_info, Mapping) else None
1805 # Always override with router config values (they take precedence over URL values)
1806 config_vertex_project: Final = litellm_params.get("vertex_project")
1807 config_vertex_location: Final = litellm_params.get("vertex_location")
1808 if config_vertex_project:
1809 vertex_project = config_vertex_project
1810 if config_vertex_location:
1811 vertex_location = config_vertex_location
1813 # Get the actual Vertex AI model name by stripping the provider prefix
1814 # e.g., "vertex_ai/gemini-2.0-flash-exp" -> "gemini-2.0-flash-exp"
1815 model_from_config: Final = litellm_params.get("model", "")
1816 if model_from_config:
1817 # get_llm_provider returns (model, custom_llm_provider, dynamic_api_key, api_base)
1818 # For "vertex_ai/gemini-2.0-flash-exp" it returns:
1819 # model="gemini-2.0-flash-exp", custom_llm_provider="vertex_ai"
1820 actual_model, custom_llm_provider, _, _ = get_llm_provider(model=model_from_config)
1822 # Log only non-sensitive information (model names and provider), never API keys or secrets.
1823 safe_actual_model: Final = actual_model
1824 safe_custom_llm_provider: Final = custom_llm_provider
1825 verbose_proxy_logger.debug(
1826 "get_llm_provider returned: actual_model=%s, custom_llm_provider=%s, model_id=%s",
1827 safe_actual_model,
1828 safe_custom_llm_provider,
1829 model_id,
1830 )
1832 if actual_model and model_id != actual_model:
1833 verbose_proxy_logger.debug(
1834 "Resolved router model '%s' to '%s' (provider=%s) with project=%s, location=%s",
1835 model_id,
1836 actual_model,
1837 custom_llm_provider,
1838 vertex_project,
1839 vertex_location,
1840 )
1841 encoded_endpoint = encoded_endpoint.replace(model_id, actual_model)
1842 endpoint = endpoint.replace(model_id, actual_model)
1844 return encoded_endpoint, endpoint, vertex_project, vertex_location, deployment_model_info
1845 except Exception as e:
1846 verbose_proxy_logger.debug("Error resolving vertex model from router for model %s: %s", model_id, e)
1848 return encoded_endpoint, endpoint, vertex_project, vertex_location, None
1851def _is_bedrock_agent_runtime_route(endpoint: str) -> bool:
1852 """
1853 Return True, if the endpoint should be routed to the `bedrock-agent-runtime` endpoint.
1854 """
1855 for _route in BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES:
1856 if _route in endpoint: 1856 ↛ 1857line 1856 didn't jump to line 1857 because the condition on line 1856 was never true
1857 return True
1858 return False
1861def _is_bedrock_agent_runtime_passthrough_disabled() -> bool:
1862 from litellm.proxy.proxy_server import general_settings
1864 setting: Final = general_settings.get("disable_bedrock_agent_runtime_passthrough")
1865 if isinstance(setting, str):
1866 return str_to_bool(setting) is True
1867 return setting is True
1870@router.api_route(
1871 "/assemblyai/{endpoint:path}",
1872 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
1873 tags=["AssemblyAI Pass-through", "pass-through"],
1874)
1875@router.api_route(
1876 "/eu.assemblyai/{endpoint:path}",
1877 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
1878 tags=["AssemblyAI EU Pass-through", "pass-through"],
1879)
1880async def assemblyai_proxy_route(
1881 endpoint: str,
1882 request: Request,
1883 fastapi_response: Response,
1884 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
1885):
1886 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler import (
1887 AssemblyAIPassthroughLoggingHandler,
1888 )
1890 """
1891 [Docs](https://api.assemblyai.com)
1892 """
1893 # Set base URL based on the route
1894 assembly_region: Final = AssemblyAIPassthroughLoggingHandler._get_assembly_region_from_url(url=str(request.url))
1895 base_target_url = AssemblyAIPassthroughLoggingHandler._get_assembly_base_url_from_region(region=assembly_region)
1896 encoded_endpoint = httpx.URL(endpoint).path
1897 # Ensure endpoint starts with '/' for proper URL construction
1898 if not encoded_endpoint.startswith("/"):
1899 encoded_endpoint = "/" + encoded_endpoint
1901 # Construct the full target URL using httpx, preserving any base path
1902 # prefix that the operator configured on base_target_url.
1903 base_url: Final = httpx.URL(base_target_url)
1904 updated_url: Final = base_url.copy_with(
1905 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
1906 )
1908 # Add or update query parameters
1909 assemblyai_api_key: Final = passthrough_endpoint_router.get_credentials(
1910 custom_llm_provider="assemblyai",
1911 region_name=assembly_region,
1912 )
1914 ## check for streaming
1915 is_streaming_request = False
1916 # assemblyai is streaming when 'stream' = True is in the body
1917 if request.method == "POST":
1918 _request_body: Final = await _json_request_body(request)
1919 if _request_body.get("stream"):
1920 is_streaming_request = True
1922 ## CREATE PASS-THROUGH
1923 endpoint_func: Final = create_pass_through_route(
1924 endpoint=endpoint,
1925 target=str(updated_url),
1926 custom_headers={"Authorization": f"{assemblyai_api_key}"},
1927 is_streaming_request=is_streaming_request,
1928 ) # dynamically construct pass-through endpoint based on incoming path
1929 received_value: Final = await endpoint_func(
1930 request=request,
1931 fastapi_response=fastapi_response,
1932 user_api_key_dict=user_api_key_dict,
1933 )
1935 return received_value
1938def get_azure_ai_search_index_from_endpoint(endpoint: str) -> str | None:
1939 """Return the index name in the ``/indexes/{name}`` position of an Azure AI
1940 Search passthrough path, or ``None`` when the path targets no index.
1942 Only the segment immediately after ``indexes`` is the operable target. Any
1943 other segment (for example the trailing ``index`` in ``.../docs/index``) must
1944 never be treated as the index, otherwise a caller authorized on one index
1945 could have Azure apply the operation to a different index on the same service.
1946 """
1947 segments: Final = endpoint.split("?", 1)[0].strip("/").split("/")
1948 for position, segment in enumerate(segments):
1949 if segment == "indexes" and position + 1 < len(segments): 1949 ↛ 1950line 1949 didn't jump to line 1950 because the condition on line 1949 was never true
1950 return segments[position + 1] or None
1951 return None
1954def is_azure_ai_search_service_level_index_create(method: str, endpoint: str) -> bool:
1955 """Return True for ``POST /indexes``, Azure AI Search's service-level index create.
1957 No index name appears in that path, so ``get_azure_ai_search_index_from_endpoint``
1958 yields None and the managed-index branch can never claim the request. Without an
1959 explicit guard it reaches the generic Azure passthrough on the proxy's own
1960 credential, so a non-admin could create an index whenever ``AZURE_API_BASE``
1961 points at the Search service.
1962 """
1963 if method != "POST":
1964 return False
1965 path: Final = endpoint.split("?", 1)[0].strip("/")
1966 return path == "indexes" or path.endswith("/indexes")
1969async def _relay_upstream_bytes(upstream: AsyncGenerator[bytes, bytes]) -> AsyncGenerator[bytes, None]:
1970 try:
1971 async for chunk in upstream:
1972 yield chunk
1973 finally:
1974 await upstream.aclose()
1977async def _relay_upstream_response(upstream: httpx.Response) -> Response:
1978 return Response(
1979 content=await upstream.aread(),
1980 status_code=upstream.status_code,
1981 headers=HttpPassThroughEndpointHelpers.get_response_headers(headers=upstream.headers, custom_headers=None),
1982 )
1985async def _relay_azure_router_model(
1986 llm_router: litellm.Router,
1987 model: str,
1988 endpoint: str,
1989 request: Request,
1990 request_body: Mapping[str, object],
1991 is_streaming_request: bool,
1992 user_api_key_dict: UserAPIKeyAuth,
1993) -> Response:
1994 foreign_deployment: Final = foreign_azure_deployment(
1995 endpoint, model, lambda: _models_served_by_group(llm_router, model)
1996 )
1997 if foreign_deployment is not None:
1998 rejection: Final[RelayRejection] = {
1999 "error": f"deployment '{foreign_deployment}' in the path is not served by model group '{model}'; "
2000 "put the model group name in the deployments segment"
2001 }
2002 raise HTTPException(status_code=400, detail=rejection)
2003 return await _relay_router_model(
2004 llm_router=llm_router,
2005 model=model,
2006 endpoint=endpoint,
2007 request=request,
2008 request_body=request_body,
2009 is_streaming_request=is_streaming_request,
2010 user_api_key_dict=user_api_key_dict,
2011 )
2014async def _relay_router_model(
2015 llm_router: litellm.Router,
2016 model: str,
2017 endpoint: str,
2018 request: Request,
2019 request_body: Mapping[str, object],
2020 is_streaming_request: bool,
2021 user_api_key_dict: UserAPIKeyAuth,
2022) -> Response:
2023 try:
2024 result: Final = await llm_router.allm_passthrough_route(
2025 model=model,
2026 method=request.method,
2027 endpoint=endpoint,
2028 request_query_params=request.query_params,
2029 request_headers=_safe_get_request_headers(request),
2030 stream=is_streaming_request,
2031 content=None,
2032 data=None,
2033 files=None,
2034 json=(request_body if is_json_content_type(request.headers.get("content-type", "")) else None),
2035 params=None,
2036 headers=None,
2037 cookies=None,
2038 litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict),
2039 )
2040 except httpx.HTTPStatusError as upstream_error:
2041 return await _relay_upstream_response(upstream_error.response)
2043 if not is_streaming_request:
2044 return await _relay_upstream_response(cast(httpx.Response, result))
2046 if inspect.isasyncgen(result):
2047 sse_headers: Final = {"content-type": "text/event-stream"}
2048 return StreamingResponse(
2049 content=wrap_passthrough_sse_bytes_with_keepalive_pings(
2050 stream=_relay_upstream_bytes(result),
2051 ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
2052 upstream_headers=sse_headers,
2053 ),
2054 status_code=200,
2055 headers=sse_headers,
2056 )
2058 upstream_stream: Final = cast(AsyncPassthroughStreamingResponse, result)
2059 return StreamingResponse(
2060 content=wrap_passthrough_sse_bytes_with_keepalive_pings(
2061 stream=_relay_upstream_bytes(upstream_stream),
2062 ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
2063 upstream_headers=upstream_stream.headers,
2064 ),
2065 status_code=upstream_stream.status_code,
2066 headers=HttpPassThroughEndpointHelpers.get_response_headers(
2067 headers=upstream_stream.headers, custom_headers=None
2068 ),
2069 )
2072@router.api_route(
2073 "/nvidia_nim/{endpoint:path}",
2074 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
2075 tags=["NVIDIA NIM Pass-through", "pass-through"],
2076)
2077async def nvidia_nim_proxy_route(
2078 endpoint: str,
2079 request: Request,
2080 fastapi_response: Response,
2081 user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
2082):
2083 """
2084 Relay a native NVIDIA NIM request through a LiteLLM model group.
2086 `{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's
2087 `api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through
2088 virtual key auth, model access checks, and spend logging.
2089 """
2090 from litellm.proxy.proxy_server import llm_router
2092 return await relay_nvidia_nim_request(
2093 llm_router=llm_router,
2094 endpoint=endpoint,
2095 request=request,
2096 request_body=await get_request_body(request),
2097 user_api_key_dict=user_api_key_dict,
2098 )
2101async def relay_nvidia_nim_request(
2102 llm_router: litellm.Router | None,
2103 endpoint: str,
2104 request: Request,
2105 request_body: Mapping[str, object],
2106 user_api_key_dict: UserAPIKeyAuth,
2107) -> Response:
2108 model_group: Final = nvidia_nim_model_group_in_path(endpoint, llm_router.get_model_list()) if llm_router else None
2109 if llm_router is None or model_group is None: 2109 ↛ 2116line 2109 didn't jump to line 2116 because the condition on line 2109 was always true
2110 rejection: Final[RelayRejection] = {
2111 "error": "no NVIDIA NIM model group in the path; call /nvidia_nim/{model_group}/v1/infer with a model "
2112 "group from your `model_list` whose deployments all use `nvidia_nim/` models"
2113 }
2114 raise HTTPException(status_code=400, detail=rejection)
2116 is_streaming_request: Final = is_passthrough_request_streaming(request_body)
2117 return await open_sse_before_first_byte(
2118 _relay_router_model(
2119 llm_router=llm_router,
2120 model=model_group,
2121 endpoint=endpoint,
2122 request=request,
2123 request_body=request_body,
2124 is_streaming_request=is_streaming_request,
2125 user_api_key_dict=user_api_key_dict,
2126 ),
2127 ping_interval_seconds=(litellm.sse_keepalive_ping_interval_seconds if is_streaming_request else None),
2128 )
2131@router.api_route(
2132 "/azure_ai/{endpoint:path}",
2133 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
2134 tags=["Azure AI Pass-through", "pass-through"],
2135)
2136@router.api_route(
2137 "/azure/{endpoint:path}",
2138 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
2139 tags=["Azure Pass-through", "pass-through"],
2140)
2141async def azure_proxy_route(
2142 endpoint: str,
2143 request: Request,
2144 fastapi_response: Response,
2145 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
2146):
2147 """
2148 Call any azure endpoint using the proxy.
2150 Just use `{PROXY_BASE_URL}/azure/{endpoint:path}`
2152 Checks if the deployment id in the url is a litellm model name. If so, it will route using the llm_router.allm_passthrough_route.
2153 """
2154 from litellm.proxy.proxy_server import llm_router
2156 if is_azure_ai_search_service_level_index_create(method=request.method, endpoint=endpoint): 2156 ↛ 2157line 2156 didn't jump to line 2157 because the condition on line 2156 was never true
2157 assert_proxy_admin_for_vector_store_index_management(user_api_key_dict, operation="create")
2159 parts: Final = endpoint.split(
2160 "/"
2161 ) # azure model is in the url - e.g. https://{endpoint}/openai/deployments/{deployment-id}/completions?api-version=2024-10-21
2163 search_index_name: Final = get_azure_ai_search_index_from_endpoint(endpoint)
2165 if len(parts) > 1 and llm_router: 2165 ↛ 2166line 2165 didn't jump to line 2166 because the condition on line 2165 was never true
2166 for part in parts:
2167 # check if LLM MODEL
2168 is_router_model = is_passthrough_request_using_router_model(
2169 request_body={"model": part}, llm_router=llm_router
2170 )
2171 # check if vector store index
2172 is_vector_store_index = (
2173 part == search_index_name
2174 and litellm.vector_store_index_registry is not None
2175 and litellm.vector_store_index_registry.is_vector_store_index(vector_store_index_name=part)
2176 )
2178 if is_router_model:
2179 request_body = await get_request_body(request)
2180 is_streaming_request = is_passthrough_request_streaming(request_body)
2181 return await open_sse_before_first_byte(
2182 _relay_azure_router_model(
2183 llm_router=llm_router,
2184 model=part,
2185 endpoint=endpoint,
2186 request=request,
2187 request_body=request_body,
2188 is_streaming_request=is_streaming_request,
2189 user_api_key_dict=user_api_key_dict,
2190 ),
2191 ping_interval_seconds=(
2192 litellm.sse_keepalive_ping_interval_seconds if is_streaming_request else None
2193 ),
2194 )
2195 elif is_vector_store_index:
2196 # get the api key from the provider config
2197 provider_config = ProviderConfigManager.get_provider_vector_stores_config(
2198 provider=litellm.LlmProviders.AZURE_AI
2199 )
2200 if provider_config is None:
2201 raise Exception("Provider config not found for Azure AI")
2202 # get the index from registry
2203 if litellm.vector_store_registry is None:
2204 raise Exception("Vector store registry not found")
2206 is_allowed_to_call_vector_store_endpoint(
2207 index_name=part,
2208 provider=litellm.LlmProviders.AZURE_AI,
2209 request=request,
2210 user_api_key_dict=user_api_key_dict,
2211 )
2212 # get the vector store name from index registry
2213 index_object = (
2214 (litellm.vector_store_index_registry.get_vector_store_index_by_name(vector_store_index_name=part))
2215 if litellm.vector_store_index_registry is not None
2216 else None
2217 )
2218 if index_object is None:
2219 raise Exception(f"Vector store index not found for {part}")
2221 vector_store_name = index_object.litellm_params.vector_store_name
2223 vector_store = litellm.vector_store_registry.get_litellm_managed_vector_store_from_registry_by_name(
2224 vector_store_name=vector_store_name
2225 )
2226 if vector_store is None:
2227 raise Exception(f"Vector store not found for {vector_store_name}")
2228 await assert_user_can_access_vector_store(
2229 vector_store=vector_store,
2230 user_api_key_dict=user_api_key_dict,
2231 )
2232 litellm_params = vector_store.get("litellm_params") or {}
2233 auth_credentials = provider_config.get_auth_credentials(litellm_params=litellm_params)
2235 extra_headers = auth_credentials.get("headers") or {}
2237 base_target_url = _optional_str(litellm_params.get("api_base"))
2238 if base_target_url is None:
2239 raise Exception(f"API base not found for {part}")
2240 return await BaseOpenAIPassThroughHandler._base_openai_pass_through_handler(
2241 endpoint=endpoint,
2242 request=request,
2243 fastapi_response=fastapi_response,
2244 user_api_key_dict=user_api_key_dict,
2245 base_target_url=base_target_url,
2246 api_key=None,
2247 custom_llm_provider=litellm.LlmProviders.AZURE_AI,
2248 extra_headers=cast(dict, extra_headers),
2249 )
2251 base_target_url = get_secret_str(secret_name="AZURE_API_BASE")
2252 if base_target_url is None: 2252 ↛ 2255line 2252 didn't jump to line 2255 because the condition on line 2252 was always true
2253 raise Exception("Required 'AZURE_API_BASE' in environment to make pass-through calls to Azure.")
2254 # Add or update query parameters
2255 azure_api_key: Final = passthrough_endpoint_router.get_credentials(
2256 custom_llm_provider=litellm.LlmProviders.AZURE.value,
2257 region_name=None,
2258 )
2259 if azure_api_key is None:
2260 raise Exception("Required 'AZURE_API_KEY' in environment to make pass-through calls to Azure.")
2262 return await BaseOpenAIPassThroughHandler._base_openai_pass_through_handler(
2263 endpoint=endpoint,
2264 request=request,
2265 fastapi_response=fastapi_response,
2266 user_api_key_dict=user_api_key_dict,
2267 base_target_url=base_target_url,
2268 api_key=azure_api_key,
2269 custom_llm_provider=litellm.LlmProviders.AZURE,
2270 )
2273from abc import ABC, abstractmethod
2275_VERTEX_LOCATION_REQUIRED_DETAIL: Final = (
2276 "No Vertex AI location for this request. Include /projects/<project>/locations/<location>/ in the "
2277 "route, set vertex_location in default_vertex_config (or DEFAULT_VERTEXAI_LOCATION), or add the "
2278 "model to model_list with use_in_pass_through: true."
2279)
2282class BaseVertexAIPassThroughHandler(ABC):
2283 @staticmethod
2284 @abstractmethod
2285 def get_default_base_target_url(vertex_location: str | None) -> str:
2286 pass
2289class VertexAIDiscoveryPassThroughHandler(BaseVertexAIPassThroughHandler):
2290 @staticmethod
2291 def get_default_base_target_url(vertex_location: str | None) -> str:
2292 return "https://discoveryengine.googleapis.com/"
2295class VertexAIPassThroughHandler(BaseVertexAIPassThroughHandler):
2296 @staticmethod
2297 def get_default_base_target_url(vertex_location: str | None) -> str:
2298 if vertex_location is None:
2299 raise HTTPException(status_code=400, detail=_VERTEX_LOCATION_REQUIRED_DETAIL)
2300 return get_vertex_base_url(vertex_location)
2303def get_vertex_base_url(vertex_location: str | None) -> str:
2304 """
2305 Base URL for Vertex AI pass-through (trailing slash for URL joining).
2307 Keep location rules aligned with ``litellm.llms.vertex_ai.common_utils.get_vertex_base_url``.
2308 """
2309 if vertex_location == "global":
2310 return "https://aiplatform.googleapis.com/"
2311 if vertex_location is None:
2312 raise ValueError("vertex_location is required")
2313 if not re.match(r"^[a-z][a-z0-9-]*$", vertex_location):
2314 raise ValueError("Invalid vertex_location format")
2315 if "-" not in vertex_location:
2316 return f"https://aiplatform.{vertex_location}.rep.googleapis.com/"
2317 return f"https://{vertex_location}-aiplatform.googleapis.com/"
2320def get_vertex_ai_allowed_incoming_headers(request: Request) -> dict:
2321 """
2322 Extract only the allowed headers from incoming request for Vertex AI pass-through.
2324 Uses an allowlist approach for security - only forwards headers we explicitly trust.
2325 This prevents accidentally forwarding sensitive headers like the LiteLLM auth token.
2327 Args:
2328 request: The FastAPI request object
2330 Returns:
2331 dict: Headers dictionary with only allowed headers
2332 """
2333 incoming_headers: Final = _safe_get_request_headers(request)
2334 headers: Final = {}
2335 for header_name in ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS:
2336 if header_name in incoming_headers:
2337 headers[header_name] = incoming_headers[header_name]
2338 return headers
2341def _is_vertex_anthropic_count_tokens_route(endpoint: str) -> bool:
2342 return endpoint.rsplit("/", 1)[-1].split(":", 1)[0] == "count-tokens"
2345def _upstream_headers_for_vertex_route(endpoint: str, headers: Mapping[str, str]) -> Mapping[str, str]:
2346 if not _is_vertex_anthropic_count_tokens_route(endpoint):
2347 return headers
2348 return MappingProxyType({name: value for name, value in headers.items() if name.lower() != "anthropic-beta"})
2351def get_vertex_pass_through_handler(
2352 call_type: Literal["discovery", "aiplatform"],
2353) -> BaseVertexAIPassThroughHandler:
2354 if call_type == "discovery":
2355 return VertexAIDiscoveryPassThroughHandler()
2356 elif call_type == "aiplatform": 2356 ↛ 2359line 2356 didn't jump to line 2359 because the condition on line 2356 was always true
2357 return VertexAIPassThroughHandler()
2358 else:
2359 raise ValueError(f"Invalid call type: {call_type}")
2362def _override_vertex_params_from_router_credentials(
2363 router_credentials: LiteLLM_ManagedVectorStore | None,
2364 vertex_project: str | None,
2365 vertex_location: str | None,
2366) -> tuple[str | None, str | None]:
2367 """
2368 Override vertex_project and vertex_location with values from router_credentials if available.
2370 Args:
2371 router_credentials: Optional vector store credentials from registry (LiteLLM_ManagedVectorStore)
2372 vertex_project: Current vertex project ID (from URL)
2373 vertex_location: Current vertex location (from URL)
2375 Returns:
2376 tuple of (vertex_project, vertex_location) with overridden values if applicable
2377 """
2378 if router_credentials is None: 2378 ↛ 2381line 2378 didn't jump to line 2381 because the condition on line 2378 was always true
2379 return vertex_project, vertex_location
2381 verbose_proxy_logger.debug("Using vector store credentials to override vertex project and location")
2383 litellm_params: Final = _string_keyed_mapping(router_credentials.get("litellm_params"))
2384 if not litellm_params:
2385 verbose_proxy_logger.warning("Vector store credentials found but litellm_params is empty")
2386 return vertex_project, vertex_location
2388 # Extract vertex_project and vertex_location from litellm_params
2389 vector_store_project: Final = _optional_str(litellm_params.get("vertex_project"))
2390 vector_store_location: Final = _optional_str(litellm_params.get("vertex_location"))
2392 if vector_store_project:
2393 verbose_proxy_logger.debug(
2394 "Overriding vertex_project from URL (%s) with vector store value: %s",
2395 vertex_project,
2396 vector_store_project,
2397 )
2398 else:
2399 verbose_proxy_logger.warning("Vector store credentials found but missing vertex_project in litellm_params")
2401 if vector_store_location:
2402 verbose_proxy_logger.debug(
2403 "Overriding vertex_location from URL (%s) with vector store value: %s",
2404 vertex_location,
2405 vector_store_location,
2406 )
2407 else:
2408 verbose_proxy_logger.warning("Vector store credentials found but missing vertex_location in litellm_params")
2410 return vector_store_project or vertex_project, vector_store_location or vertex_location
2413_CREDENTIALLESS_VERTEX_MISSING_CREDENTIAL_DETAIL: Final = (
2414 "No Vertex AI credential is configured on this proxy and the request carried no upstream "
2415 "Google credential. The LiteLLM virtual key is not forwarded to Google. Configure a Vertex "
2416 "credential (DEFAULT_VERTEXAI_PROJECT / DEFAULT_VERTEXAI_LOCATION / DEFAULT_VERTEXAI_CREDENTIALS, "
2417 "or a model with use_in_pass_through: true), or send your own Google OAuth token in the "
2418 "Authorization header."
2419)
2422def _normalize_credential_value(value: str) -> str:
2423 """Reduce a header value to the bare token, matching how ``user_api_key_auth``
2424 reads a caller's key.
2426 Reuses the auth module's ``_get_bearer_token`` so the caller-key comparison
2427 strips exactly the schemes authentication accepts (``Bearer`` / ``bearer`` /
2428 ``Basic`` / ``AWS4-HMAC-SHA256`` credential), rather than re-deriving a
2429 narrower normalization here. ``_get_bearer_token`` returns ``""`` for a value
2430 with no recognized scheme prefix, so a bare token (or a real Google
2431 credential that carries no scheme) falls back to its own value.
2432 """
2433 return _get_bearer_token(value) or value
2436_VERTEX_UPSTREAM_CREDENTIAL_HEADERS: Final = frozenset({"authorization", "x-goog-api-key"})
2437_HEADERS_NEVER_FORWARDED_TO_VERTEX: Final = frozenset({"content-length", "host"}) | (
2438 SpecialHeaders.litellm_credential_header_names() - _VERTEX_UPSTREAM_CREDENTIAL_HEADERS
2439)
2441_CREDENTIALLESS_ANTHROPIC_MISSING_CREDENTIAL_DETAIL: Final = (
2442 "No Anthropic credential is configured on this proxy and the request carried no upstream "
2443 "Anthropic credential. The LiteLLM virtual key is not forwarded to Anthropic. Configure an "
2444 "Anthropic credential (ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN, or a model with "
2445 "use_in_pass_through: true), or send your own Anthropic API key in the x-api-key header or "
2446 "your own Anthropic OAuth token in the Authorization header."
2447)
2449_ANTHROPIC_UPSTREAM_CREDENTIAL_HEADERS: Final = frozenset({"authorization", "x-api-key"})
2450_HEADERS_NEVER_FORWARDED_TO_ANTHROPIC: Final = frozenset({"content-length", "host", "accept-encoding"}) | (
2451 SpecialHeaders.litellm_credential_header_names() - _ANTHROPIC_UPSTREAM_CREDENTIAL_HEADERS
2452)
2453_HEADERS_NEVER_FORWARDED_TO_BEDROCK: Final = (
2454 frozenset({"content-length", "host", "accept-encoding"}) | SpecialHeaders.litellm_credential_header_names()
2455)
2458_MAPPED_ROUTE_CALLER_KEY_HEADER: Final = "litellm_user_api_key"
2461def _operator_configured_caller_key_header_names() -> tuple[str, ...]:
2462 """Operator-configured header names ``user_api_key_auth`` reads the caller's key from."""
2463 from litellm.proxy.proxy_server import general_settings
2465 custom_key_header: Final = general_settings.get("litellm_key_header_name")
2466 override: Final = (custom_key_header.lower(),) if isinstance(custom_key_header, str) else ()
2467 pass_through_endpoints: Final = general_settings.get("pass_through_endpoints")
2468 endpoints: Final = pass_through_endpoints if isinstance(pass_through_endpoints, list) else ()
2469 pass_through: Final = tuple(
2470 dict.fromkeys(
2471 headers["litellm_user_api_key"].lower()
2472 for endpoint in endpoints
2473 if isinstance(endpoint, dict)
2474 for headers in (endpoint.get("headers"),)
2475 if isinstance(headers, dict) and isinstance(headers.get("litellm_user_api_key"), str)
2476 )
2477 )
2478 return override + pass_through
2481def _is_authenticated_caller_jwt(value: str, jwt_claims: Mapping[str, object]) -> bool:
2482 """Whether a header value is the JWT whose claims ``user_api_key_auth`` stored as ``jwt_claims``."""
2483 presented_claims: Final = JWTHandler.get_unverified_claims(value)
2484 if presented_claims is None:
2485 return False
2486 return all(
2487 presented_claims.get(name) == claim
2488 for name, claim in jwt_claims.items()
2489 if name not in JWTHandler.LITELLM_INTERNAL_CLAIMS
2490 )
2493def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAuth) -> bool:
2494 """Whether a header value is the master key, the JWT that authenticated, or the key stored as ``api_key``.
2496 A proxy in no-auth dev mode without custom auth authenticated nothing, so none of the caller's values is one.
2497 """
2498 from litellm.proxy.proxy_server import general_settings, master_key, user_custom_auth
2500 normalized: Final = _normalize_credential_value(value)
2501 if master_key is not None and hmac.compare_digest(normalized.encode(), master_key.encode()): 2501 ↛ 2502line 2501 didn't jump to line 2502 because the condition on line 2501 was never true
2502 return True
2503 jwt_claims: Final = user_api_key_dict.jwt_claims
2504 if jwt_claims and _is_authenticated_caller_jwt(normalized, jwt_claims): 2504 ↛ 2505line 2504 didn't jump to line 2505 because the condition on line 2504 was never true
2505 return True
2506 if is_no_auth_dev_mode(master_key, general_settings) and user_custom_auth is None: 2506 ↛ 2507line 2506 didn't jump to line 2507 because the condition on line 2506 was never true
2507 return False
2508 authenticated_key: Final = user_api_key_dict.api_key
2509 if authenticated_key is None: 2509 ↛ 2510line 2509 didn't jump to line 2510 because the condition on line 2509 was never true
2510 return False
2511 stored_representation: Final = UserAPIKeyAuth._safe_hash_litellm_api_key(normalized) # pyright: ignore[reportPrivateUsage] # the exact transform auth applied when it stored api_key
2512 return hmac.compare_digest(stored_representation.encode(), authenticated_key.encode())
2515def _caller_headers_without_litellm_secrets(
2516 request: Request, user_api_key_dict: UserAPIKeyAuth, never_forwarded: frozenset[str]
2517) -> Mapping[str, str]:
2518 incoming: Final = _safe_get_request_headers(request)
2519 dropped_by_name: Final = never_forwarded.union(
2520 (_MAPPED_ROUTE_CALLER_KEY_HEADER, *_operator_configured_caller_key_header_names())
2521 )
2522 return MappingProxyType(
2523 {
2524 name: value
2525 for name, value in incoming.items()
2526 if name not in dropped_by_name and not _is_authenticated_caller_secret(value, user_api_key_dict)
2527 }
2528 )
2531def _forwarded_headers_for_credentialless_vertex_passthrough(
2532 request: Request, user_api_key_dict: UserAPIKeyAuth
2533) -> Mapping[str, str]:
2534 """Caller headers to forward on the bring-your-own-credentials Vertex branch, minus LiteLLM secrets."""
2535 forwarded: Final = _caller_headers_without_litellm_secrets(
2536 request, user_api_key_dict, _HEADERS_NEVER_FORWARDED_TO_VERTEX
2537 )
2538 if _VERTEX_UPSTREAM_CREDENTIAL_HEADERS.isdisjoint(forwarded): 2538 ↛ 2540line 2538 didn't jump to line 2540 because the condition on line 2538 was always true
2539 raise HTTPException(status_code=401, detail=_CREDENTIALLESS_VERTEX_MISSING_CREDENTIAL_DETAIL)
2540 return forwarded
2543def _upstream_headers_for_anthropic_route(
2544 request: Request, user_api_key_dict: UserAPIKeyAuth, proxy_auth_header: Mapping[str, str] | None
2545) -> Mapping[str, str]:
2546 caller_headers: Final = _caller_headers_without_litellm_secrets(
2547 request, user_api_key_dict, _HEADERS_NEVER_FORWARDED_TO_ANTHROPIC
2548 )
2549 if proxy_auth_header is None and _ANTHROPIC_UPSTREAM_CREDENTIAL_HEADERS.isdisjoint(caller_headers): 2549 ↛ 2551line 2549 didn't jump to line 2551 because the condition on line 2549 was always true
2550 raise HTTPException(status_code=401, detail=_CREDENTIALLESS_ANTHROPIC_MISSING_CREDENTIAL_DETAIL)
2551 return MappingProxyType({**caller_headers, **(proxy_auth_header or {})})
2554def _upstream_headers_for_bedrock_agent_runtime_route(
2555 request: Request, user_api_key_dict: UserAPIKeyAuth, signed_headers: Mapping[str, object]
2556) -> Mapping[str, object]:
2557 caller_headers: Final = _caller_headers_without_litellm_secrets(
2558 request,
2559 user_api_key_dict,
2560 _HEADERS_NEVER_FORWARDED_TO_BEDROCK | frozenset(name.lower() for name in signed_headers),
2561 )
2562 return MappingProxyType({**caller_headers, **signed_headers})
2565async def _prepare_vertex_auth_headers(
2566 request: Request,
2567 vertex_credentials: VertexPassThroughCredentials | None,
2568 router_credentials: LiteLLM_ManagedVectorStore | None,
2569 vertex_project: str | None,
2570 vertex_location: str | None,
2571 user_api_key_dict: UserAPIKeyAuth,
2572) -> tuple[Mapping[str, str], bool, str | None, str | None]:
2573 """
2574 Prepare authentication headers for Vertex AI pass-through requests.
2576 Args:
2577 request: FastAPI request object
2578 vertex_credentials: Vertex AI credentials from config
2579 router_credentials: Optional vector store credentials from registry
2580 vertex_project: Vertex project ID
2581 vertex_location: Vertex location
2582 user_api_key_dict: The caller's resolved authentication, so only the secret that
2583 authenticated them is stripped on the credential-less branch
2585 Returns:
2586 tuple containing:
2587 - headers: dict - Authentication headers to use
2588 - headers_passed_through: bool - Whether headers were passed through from request
2589 - vertex_project: str | None - Updated vertex project ID
2590 - vertex_location: str | None - Updated vertex location
2591 """
2592 vertex_llm_base: Final = VertexBase()
2593 headers_passed_through = False
2595 # Use headers from the incoming request if no vertex credentials are found
2596 if (vertex_credentials is None or vertex_credentials.vertex_project is None) and router_credentials is None: 2596 ↛ 2603line 2596 didn't jump to line 2603 because the condition on line 2596 was always true
2597 headers = _forwarded_headers_for_credentialless_vertex_passthrough(request, user_api_key_dict)
2598 headers_passed_through = True
2599 verbose_proxy_logger.debug(
2600 "default_vertex_config not set, forwarding caller-provided headers %s", tuple(headers.keys())
2601 )
2602 else:
2603 if router_credentials is not None:
2604 vertex_credentials_str = None
2605 elif vertex_credentials is not None:
2606 # Use credentials from vertex_credentials
2607 # When vertex_credentials are provided (including default credentials),
2608 # use their project/location values if available
2609 if vertex_credentials.vertex_project is not None:
2610 vertex_project = vertex_credentials.vertex_project
2611 if vertex_credentials.vertex_location is not None:
2612 vertex_location = vertex_credentials.vertex_location
2613 vertex_credentials_str = vertex_credentials.vertex_credentials
2614 else:
2615 raise ValueError("No vertex credentials found")
2617 _auth_header, vertex_project = await vertex_llm_base._ensure_access_token_async(
2618 credentials=vertex_credentials_str,
2619 project_id=vertex_project,
2620 custom_llm_provider="vertex_ai_beta",
2621 )
2623 auth_header, _ = vertex_llm_base._get_token_and_url(
2624 model="",
2625 auth_header=_auth_header,
2626 gemini_api_key=None,
2627 vertex_credentials=vertex_credentials_str,
2628 vertex_project=vertex_project,
2629 vertex_location=vertex_location,
2630 stream=False,
2631 custom_llm_provider="vertex_ai_beta",
2632 api_base="",
2633 )
2635 # Use allowlist approach - only forward specific safe headers
2636 headers = get_vertex_ai_allowed_incoming_headers(request)
2637 # Add the Authorization header with vendor credentials
2638 headers["Authorization"] = f"Bearer {auth_header}"
2640 return (
2641 headers,
2642 headers_passed_through,
2643 vertex_project,
2644 vertex_location,
2645 )
2648async def _base_vertex_proxy_route(
2649 endpoint: str,
2650 request: Request,
2651 fastapi_response: Response,
2652 get_vertex_pass_through_handler: BaseVertexAIPassThroughHandler,
2653 user_api_key_dict: UserAPIKeyAuth | None = None,
2654 router_credentials: LiteLLM_ManagedVectorStore | None = None,
2655):
2656 """
2657 Base function for Vertex AI passthrough routes.
2658 Handles common logic for all Vertex AI services.
2660 Default base_target_url is derived from ``get_vertex_base_url`` in this module
2661 (regional, ``global``, or multi-region ``.rep.`` hosts), with a trailing slash.
2663 Args:
2664 endpoint: The endpoint path
2665 request: FastAPI request object
2666 fastapi_response: FastAPI response object
2667 get_vertex_pass_through_handler: Handler for the specific Vertex AI service
2668 user_api_key_dict: User API key authentication dict
2669 router_credentials: Optional vector store credentials from registry (LiteLLM_ManagedVectorStore)
2670 """
2671 from litellm.llms.vertex_ai.common_utils import (
2672 construct_target_url,
2673 get_vertex_location_from_url,
2674 get_vertex_model_id_from_url,
2675 get_vertex_project_id_from_url,
2676 )
2677 from litellm.proxy.proxy_server import llm_router
2679 encoded_endpoint = httpx.URL(endpoint).path
2680 verbose_proxy_logger.debug("requested endpoint %s", endpoint)
2681 headers: Mapping[str, str] = {}
2682 api_key_to_use = get_litellm_virtual_key(request=request)
2683 user_api_key_dict = await user_api_key_auth(
2684 request=request,
2685 api_key=api_key_to_use,
2686 )
2688 if user_api_key_dict is None: 2688 ↛ 2689line 2688 didn't jump to line 2689 because the condition on line 2688 was never true
2689 api_key_to_use = get_litellm_virtual_key(request=request)
2690 user_api_key_dict = await user_api_key_auth(
2691 request=request,
2692 api_key=api_key_to_use,
2693 )
2694 if router_credentials is not None: 2694 ↛ 2695line 2694 didn't jump to line 2695 because the condition on line 2694 was never true
2695 await assert_user_can_access_vector_store(
2696 vector_store=router_credentials,
2697 user_api_key_dict=user_api_key_dict,
2698 )
2700 vertex_project: str | None = get_vertex_project_id_from_url(endpoint)
2701 vertex_location: str | None = get_vertex_location_from_url(endpoint)
2703 # Override with vector store credentials if available
2704 vertex_project, vertex_location = _override_vertex_params_from_router_credentials(
2705 router_credentials=router_credentials,
2706 vertex_project=vertex_project,
2707 vertex_location=vertex_location,
2708 )
2710 # Check if model is in router config - always do this to resolve custom model names
2711 model_id: Final = get_vertex_model_id_from_url(endpoint)
2712 if model_id: 2712 ↛ 2713line 2712 didn't jump to line 2713 because the condition on line 2712 was never true
2713 if llm_router:
2714 # Resolve model configuration from router
2715 (
2716 encoded_endpoint,
2717 endpoint,
2718 vertex_project,
2719 vertex_location,
2720 deployment_model_info,
2721 ) = _resolve_vertex_model_from_router(
2722 model_id=model_id,
2723 llm_router=llm_router,
2724 encoded_endpoint=encoded_endpoint,
2725 endpoint=endpoint,
2726 vertex_project=vertex_project,
2727 vertex_location=vertex_location,
2728 )
2729 if deployment_model_info:
2730 setattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, deployment_model_info)
2732 vertex_credentials: Final = passthrough_endpoint_router.get_vertex_credentials(
2733 project_id=vertex_project,
2734 location=vertex_location,
2735 )
2737 # Prepare authentication headers
2738 (
2739 headers,
2740 headers_passed_through,
2741 vertex_project,
2742 vertex_location,
2743 ) = await _prepare_vertex_auth_headers(
2744 request=request,
2745 vertex_credentials=vertex_credentials,
2746 router_credentials=router_credentials,
2747 vertex_project=vertex_project,
2748 vertex_location=vertex_location,
2749 user_api_key_dict=user_api_key_dict,
2750 )
2752 base_target_url: Final = get_vertex_pass_through_handler.get_default_base_target_url(vertex_location)
2754 request_route: Final = encoded_endpoint
2755 verbose_proxy_logger.debug("request_route %s", request_route)
2757 # Ensure endpoint starts with '/' for proper URL construction
2758 if not encoded_endpoint.startswith("/"):
2759 encoded_endpoint = "/" + encoded_endpoint
2761 # Construct the full target URL using httpx
2762 updated_url: Final = construct_target_url(
2763 base_url=base_target_url,
2764 requested_route=encoded_endpoint,
2765 vertex_location=vertex_location,
2766 vertex_project=vertex_project,
2767 )
2769 verbose_proxy_logger.debug("updated url %s", updated_url)
2771 ## check for streaming
2772 target = str(updated_url)
2773 if ":rawPredict" in target or ":streamRawPredict" in target:
2774 is_streaming_request = await is_streaming_request_fn(request)
2775 else:
2776 is_streaming_request = "stream" in target
2777 if is_streaming_request:
2778 target += "?alt=sse"
2780 ## CREATE PASS-THROUGH
2781 endpoint_func: Final = create_pass_through_route(
2782 endpoint=endpoint,
2783 target=target,
2784 custom_headers=_upstream_headers_for_vertex_route(endpoint, headers),
2785 is_streaming_request=is_streaming_request,
2786 ) # dynamically construct pass-through endpoint based on incoming path
2788 try:
2789 received_value: Final = await endpoint_func(
2790 request,
2791 fastapi_response,
2792 user_api_key_dict,
2793 )
2794 except ProxyException as e:
2795 if headers_passed_through:
2796 e.message = f"No credentials found on proxy for project_name={vertex_project} + location={vertex_location}, check `/model/info` for allowed project + region combinations with `use_in_pass_through: true`. Headers were passed through directly but request failed with error: {e.message}"
2797 raise e
2799 return received_value
2802@router.api_route(
2803 "/vertex_ai/discovery/{endpoint:path}",
2804 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
2805 tags=["Vertex AI Pass-through", "pass-through"],
2806)
2807async def vertex_discovery_proxy_route(
2808 endpoint: str,
2809 request: Request,
2810 fastapi_response: Response,
2811):
2812 """
2813 Call any vertex discovery endpoint using the proxy.
2815 Just use `{PROXY_BASE_URL}/vertex_ai/discovery/{endpoint:path}`
2817 Target url: `https://discoveryengine.googleapis.com`
2818 """
2819 import re
2821 # Extract vector store ID from endpoint if present (e.g., dataStores/test-litellm-app_1761094730750)
2822 vector_store_credentials: LiteLLM_ManagedVectorStore | None = None
2823 vector_store_id_match: Final = re.search(r"dataStores/([^/]+)", endpoint)
2825 if vector_store_id_match: 2825 ↛ 2826line 2825 didn't jump to line 2826 because the condition on line 2825 was never true
2826 vector_store_id: Final = vector_store_id_match.group(1)
2827 verbose_proxy_logger.debug("Extracted vector store ID from endpoint: %s", vector_store_id)
2829 # Retrieve LiteLLM-managed vector store credentials if the datastore id
2830 # is registered with LiteLLM. Unknown datastore ids keep the existing
2831 # direct Vertex pass-through behavior.
2832 vector_store_credentials = await get_litellm_managed_vector_store(vector_store_id=vector_store_id)
2834 if vector_store_credentials:
2835 verbose_proxy_logger.debug("Found vector store credentials for ID: %s", vector_store_id)
2836 else:
2837 verbose_proxy_logger.debug(
2838 "Vector store ID %s found in endpoint but no credentials found in registry",
2839 vector_store_id,
2840 )
2842 discovery_handler: Final = get_vertex_pass_through_handler(call_type="discovery")
2843 return await _base_vertex_proxy_route(
2844 endpoint=endpoint,
2845 request=request,
2846 fastapi_response=fastapi_response,
2847 get_vertex_pass_through_handler=discovery_handler,
2848 router_credentials=vector_store_credentials,
2849 )
2852@router.api_route(
2853 "/vertex-ai/{endpoint:path}",
2854 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
2855 tags=["Vertex AI Pass-through", "pass-through"],
2856 include_in_schema=False,
2857)
2858@router.api_route(
2859 "/vertex_ai/{endpoint:path}",
2860 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
2861 tags=["Vertex AI Pass-through", "pass-through"],
2862)
2863async def vertex_proxy_route(
2864 endpoint: str,
2865 request: Request,
2866 fastapi_response: Response,
2867 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
2868):
2869 """
2870 Call LiteLLM proxy via Vertex AI SDK.
2872 [Docs](https://docs.litellm.ai/docs/pass_through/vertex_ai)
2873 """
2874 ai_platform_handler: Final = get_vertex_pass_through_handler(call_type="aiplatform")
2876 return await _base_vertex_proxy_route(
2877 endpoint=endpoint,
2878 request=request,
2879 fastapi_response=fastapi_response,
2880 get_vertex_pass_through_handler=ai_platform_handler,
2881 user_api_key_dict=user_api_key_dict,
2882 )
2885@router.api_route(
2886 "/openai/{endpoint:path}",
2887 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
2888 tags=["OpenAI Pass-through", "pass-through"],
2889)
2890async def openai_proxy_route(
2891 endpoint: str,
2892 request: Request,
2893 fastapi_response: Response,
2894 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
2895):
2896 """
2897 Pass-through endpoint for OpenAI API calls.
2899 Available on both routes:
2900 - /openai/{endpoint:path} - Standard OpenAI passthrough route
2901 - /openai_passthrough/{endpoint:path} - Dedicated passthrough route (recommended for Responses API)
2903 Use /openai_passthrough/* when you need guaranteed passthrough to OpenAI without conflicts
2904 with LiteLLM's native implementations (e.g., for the Responses API at /v1/responses).
2906 Examples:
2907 Standard route:
2908 - /openai/v1/chat/completions
2909 - /openai/v1/assistants
2910 - /openai/v1/threads
2912 Dedicated passthrough (for Responses API):
2913 - /openai_passthrough/v1/responses
2914 - /openai_passthrough/v1/responses/{response_id}
2915 - /openai_passthrough/v1/responses/{response_id}/input_items
2917 [Docs](https://docs.litellm.ai/docs/pass_through/openai_passthrough)
2918 """
2919 base_target_url: Final = os.getenv("OPENAI_API_BASE") or "https://api.openai.com/"
2920 # Add or update query parameters
2921 openai_api_key: Final = passthrough_endpoint_router.get_credentials(
2922 custom_llm_provider=litellm.LlmProviders.OPENAI.value,
2923 region_name=None,
2924 )
2925 if openai_api_key is None: 2925 ↛ 2928line 2925 didn't jump to line 2928 because the condition on line 2925 was always true
2926 raise Exception("Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI.")
2928 return await BaseOpenAIPassThroughHandler._base_openai_pass_through_handler(
2929 endpoint=endpoint,
2930 request=request,
2931 fastapi_response=fastapi_response,
2932 user_api_key_dict=user_api_key_dict,
2933 base_target_url=base_target_url,
2934 api_key=openai_api_key,
2935 custom_llm_provider=litellm.LlmProviders.OPENAI,
2936 )
2939def _join_url_paths(base_url: httpx.URL, path: str, custom_llm_provider: litellm.LlmProviders) -> str:
2940 """
2941 Properly joins a base URL with a path, preserving any existing path in the base URL.
2942 """
2943 # Combine paths via the shared helper so any '..' in the path cannot
2944 # climb above the configured base path.
2945 joined_path_str = str(
2946 base_url.copy_with(path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, path))
2947 )
2949 # Apply OpenAI-specific path handling for both branches
2950 if custom_llm_provider == litellm.LlmProviders.OPENAI and "/v1/" not in joined_path_str:
2951 # Insert v1 after api.openai.com for OpenAI requests
2952 joined_path_str = joined_path_str.replace("api.openai.com/", "api.openai.com/v1/")
2954 return joined_path_str
2957_OPENAI_WS_ALL_MODEL_ACCESS: Final = frozenset(
2958 {
2959 SpecialModelNames.all_proxy_models.value,
2960 SpecialModelNames.all_team_models.value,
2961 "*",
2962 }
2963)
2966def _has_model_restrictions(model_allowlists: tuple[Sequence[str], ...]) -> bool:
2967 return any(str(model) not in _OPENAI_WS_ALL_MODEL_ACCESS for allowlist in model_allowlists for model in allowlist)
2970@dataclass(frozen=True, slots=True)
2971class _OpenAIWebsocketRefusal:
2972 close_reason: str
2973 message: str
2976class _OpenAIWebsocketErrorDetail(TypedDict):
2977 type: ReadOnly[Literal["invalid_request_error"]]
2978 message: ReadOnly[str]
2981class _OpenAIWebsocketErrorFrame(TypedDict):
2982 type: ReadOnly[Literal["error"]]
2983 error: ReadOnly[_OpenAIWebsocketErrorDetail]
2986_OPENAI_WS_DISABLED_REFUSAL: Final = _OpenAIWebsocketRefusal(
2987 close_reason="OpenAI websocket passthrough is disabled",
2988 message=(
2989 "OpenAI websocket passthrough is disabled on this gateway. A proxy admin can turn it on by "
2990 "setting general_settings.enable_openai_websocket_passthrough to true."
2991 ),
2992)
2994_OPENAI_WS_MODEL_RESTRICTED_REFUSAL: Final = _OpenAIWebsocketRefusal(
2995 close_reason="Keys with model restrictions cannot use OpenAI websocket passthrough",
2996 message=(
2997 "Keys with model restrictions cannot use OpenAI websocket passthrough, because this route "
2998 "relays frames to the provider without reading which model they ask for."
2999 ),
3000)
3003def _is_openai_websocket_passthrough_enabled(general_settings: Mapping[str, object]) -> bool:
3004 setting: Final = general_settings.get("enable_openai_websocket_passthrough")
3005 if isinstance(setting, str):
3006 return str_to_bool(setting) is True
3007 return setting is True
3010class _OpenAIWebsocketModelAllowlists(Protocol):
3011 async def __call__(self, valid_token: UserAPIKeyAuth, /) -> tuple[Sequence[str], ...]: ... 3011 ↛ exitline 3011 didn't return from function '__call__' because
3014async def _openai_websocket_refusal(
3015 user_api_key_dict: UserAPIKeyAuth,
3016 general_settings: Mapping[str, object],
3017 model_allowlists: _OpenAIWebsocketModelAllowlists,
3018) -> _OpenAIWebsocketRefusal | None:
3019 if not _is_openai_websocket_passthrough_enabled(general_settings):
3020 return _OPENAI_WS_DISABLED_REFUSAL
3021 if _has_model_restrictions(await model_allowlists(user_api_key_dict)):
3022 return _OPENAI_WS_MODEL_RESTRICTED_REFUSAL
3023 return None
3026class _WebsocketRelay(Protocol):
3027 async def __call__( 3027 ↛ exitline 3027 didn't return from function '__call__' because
3028 self,
3029 *,
3030 websocket: WebSocket,
3031 target: str,
3032 custom_headers: dict[str, str], # mutable-ok: the relay takes a plain dict of upstream headers
3033 user_api_key_dict: UserAPIKeyAuth,
3034 forward_headers: bool,
3035 endpoint: str,
3036 accept_websocket: bool,
3037 ) -> None: ...
3040def _websocket_relay() -> _WebsocketRelay:
3041 return websocket_passthrough_request
3044def _proxy_model_allowlists() -> _OpenAIWebsocketModelAllowlists:
3045 from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
3047 async def resolve(valid_token: UserAPIKeyAuth, /) -> tuple[Sequence[str], ...]:
3048 return await enforced_model_allowlists(
3049 valid_token=valid_token,
3050 prisma_client=prisma_client,
3051 user_api_key_cache=user_api_key_cache,
3052 proxy_logging_obj=proxy_logging_obj,
3053 )
3055 return resolve
3058def _negotiated_websocket_subprotocol(websocket: WebSocket) -> str | None:
3059 requested_subprotocols: Final = tuple(
3060 protocol.strip()
3061 for protocol in (websocket.headers.get("sec-websocket-protocol") or "").split(",")
3062 if protocol.strip()
3063 )
3064 return requested_subprotocols[0] if requested_subprotocols else None
3067@router.websocket("/openai_passthrough/{endpoint:path}")
3068@router.websocket("/openai/{endpoint:path}")
3069async def openai_websocket_proxy_route(
3070 websocket: WebSocket,
3071 endpoint: str,
3072 user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth_websocket)],
3073 general_settings: Annotated[Mapping[str, object], Depends(_proxy_general_settings)],
3074 relay: Annotated[_WebsocketRelay, Depends(_websocket_relay)],
3075 model_allowlists: Annotated[_OpenAIWebsocketModelAllowlists, Depends(_proxy_model_allowlists)],
3076) -> None:
3077 """WebSocket passthrough for OpenAI prefixes (realtime / responses.connect)."""
3078 negotiated_subprotocol: Final = _negotiated_websocket_subprotocol(websocket)
3080 refusal: Final = await _openai_websocket_refusal(user_api_key_dict, general_settings, model_allowlists)
3081 if refusal is not None:
3082 await websocket.accept(subprotocol=negotiated_subprotocol)
3083 error_frame: Final[_OpenAIWebsocketErrorFrame] = {
3084 "type": "error",
3085 "error": {"type": "invalid_request_error", "message": refusal.message},
3086 }
3087 await websocket.send_text(json.dumps(error_frame))
3088 await websocket.close(code=1008, reason=refusal.close_reason)
3089 return
3091 base_target_url: Final = os.getenv("OPENAI_API_BASE") or "https://api.openai.com/"
3092 openai_api_key: Final = passthrough_endpoint_router.get_credentials(
3093 custom_llm_provider=litellm.LlmProviders.OPENAI.value,
3094 region_name=None,
3095 )
3096 if openai_api_key is None:
3097 await websocket.close(
3098 code=1011,
3099 reason="Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI.",
3100 )
3101 return
3103 raw_path: Final = httpx.URL(endpoint).path
3104 encoded_endpoint: Final = raw_path if raw_path.startswith("/") else f"/{raw_path}"
3105 base_url: Final = httpx.URL(base_target_url)
3106 updated_url: Final = _join_url_paths(
3107 base_url=base_url,
3108 path=encoded_endpoint,
3109 custom_llm_provider=litellm.LlmProviders.OPENAI,
3110 )
3111 wss_base: Final = (
3112 "wss://" + updated_url[len("https://") :]
3113 if updated_url.startswith("https://")
3114 else "ws://" + updated_url[len("http://") :]
3115 if updated_url.startswith("http://")
3116 else updated_url
3117 )
3118 query_string: Final = websocket.url.query
3119 wss_target: Final = f"{wss_base}{'&' if '?' in wss_base else '?'}{query_string}" if query_string else wss_base
3120 custom_headers: Final = { # mutable-ok: websocket_passthrough_request requires a plain dict of upstream headers
3121 "Authorization": f"Bearer {openai_api_key}"
3122 }
3124 await websocket.accept(subprotocol=negotiated_subprotocol)
3126 await relay(
3127 websocket=websocket,
3128 target=wss_target,
3129 custom_headers=custom_headers,
3130 user_api_key_dict=user_api_key_dict,
3131 forward_headers=False,
3132 endpoint=websocket.url.path,
3133 accept_websocket=False,
3134 )
3137_DEEPGRAM_WS_MISSING_KEY_REASON: Final = (
3138 "Required 'DEEPGRAM_API_KEY' in environment to make pass-through calls to Deepgram."
3139)
3140_DEEPGRAM_WS_CALLBACK_REASON: Final = "Deepgram callback delivery is not supported through the proxy: remove {params}"
3141_DEEPGRAM_WS_UNPRICED_REASON: Final = (
3142 "No streaming price for '{registry_key}': add it to the model cost map to enable it"
3143)
3146async def deepgram_listen_user_api_key_auth(websocket: WebSocket) -> UserAPIKeyAuth:
3147 return await user_api_key_auth_websocket_for_model(
3148 websocket, model=deepgram_listen_requested_model(websocket.url.query)
3149 )
3152@router.websocket("/deepgram/v1/listen")
3153@router.websocket("/deepgram/listen")
3154async def deepgram_listen_websocket_route(
3155 websocket: WebSocket,
3156 user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(deepgram_listen_user_api_key_auth)],
3157 relay: Annotated[_WebsocketRelay, Depends(_websocket_relay)],
3158) -> None:
3159 deepgram_api_key: Final = passthrough_endpoint_router.get_credentials(
3160 custom_llm_provider=litellm.LlmProviders.DEEPGRAM.value,
3161 region_name=None,
3162 )
3163 if deepgram_api_key is None:
3164 await websocket.close(code=1011, reason=_DEEPGRAM_WS_MISSING_KEY_REASON)
3165 return
3167 await websocket.accept(subprotocol=_negotiated_websocket_subprotocol(websocket))
3168 callback_params: Final = deepgram_listen_callback_params(websocket.url.query)
3169 if callback_params:
3170 await websocket.close(
3171 code=1008,
3172 reason=_DEEPGRAM_WS_CALLBACK_REASON.format(params=", ".join(callback_params)),
3173 )
3174 return
3176 target: Final = deepgram_listen_websocket_target(
3177 api_base=get_secret_str("DEEPGRAM_API_BASE"),
3178 query_string=websocket.url.query,
3179 )
3180 if not deepgram_listen_is_priced(target):
3181 await websocket.close(
3182 code=1008,
3183 reason=_DEEPGRAM_WS_UNPRICED_REASON.format(registry_key=deepgram_listen_registry_key(target)),
3184 )
3185 return
3187 await relay(
3188 websocket=websocket,
3189 target=target,
3190 custom_headers={ # mutable-ok: websocket_passthrough_request requires a plain dict of upstream headers
3191 "Authorization": f"Token {deepgram_api_key}"
3192 },
3193 user_api_key_dict=user_api_key_dict,
3194 forward_headers=False,
3195 endpoint=websocket.url.path,
3196 accept_websocket=False,
3197 )
3200class BaseOpenAIPassThroughHandler:
3201 @staticmethod
3202 async def _base_openai_pass_through_handler(
3203 endpoint: str,
3204 request: Request,
3205 fastapi_response: Response,
3206 user_api_key_dict: UserAPIKeyAuth,
3207 base_target_url: str,
3208 api_key: str | None,
3209 custom_llm_provider: litellm.LlmProviders,
3210 extra_headers: dict | None = None,
3211 ):
3212 encoded_endpoint = httpx.URL(endpoint).path
3213 # Ensure endpoint starts with '/' for proper URL construction
3214 if not encoded_endpoint.startswith("/"):
3215 encoded_endpoint = "/" + encoded_endpoint
3217 # Construct the full target URL by properly joining the base URL and endpoint path
3218 base_url: Final = httpx.URL(base_target_url)
3219 updated_url: Final = _join_url_paths(
3220 base_url=base_url,
3221 path=encoded_endpoint,
3222 custom_llm_provider=custom_llm_provider,
3223 )
3225 ## check for streaming
3226 is_streaming_request = False
3227 if "stream" in str(updated_url):
3228 is_streaming_request = True
3230 ## CREATE PASS-THROUGH
3231 endpoint_func: Final = create_pass_through_route(
3232 endpoint=endpoint,
3233 target=str(updated_url),
3234 custom_headers=BaseOpenAIPassThroughHandler._assemble_headers(
3235 api_key=api_key, request=request, extra_headers=extra_headers
3236 ),
3237 is_streaming_request=is_streaming_request,
3238 custom_llm_provider=(
3239 custom_llm_provider.value
3240 if hasattr(custom_llm_provider, "value")
3241 else str(custom_llm_provider)
3242 if custom_llm_provider
3243 else None
3244 ),
3245 ) # dynamically construct pass-through endpoint based on incoming path
3246 received_value: Final = await endpoint_func(
3247 request,
3248 fastapi_response,
3249 user_api_key_dict,
3250 )
3252 return received_value
3254 @staticmethod
3255 def _append_openai_beta_header(headers: dict, request: Request) -> dict:
3256 """
3257 Appends the OpenAI-Beta header to the headers if the request is an OpenAI Assistants API request
3258 """
3259 if RouteChecks._is_assistants_api_request(request) is True and "OpenAI-Beta" not in headers:
3260 headers["OpenAI-Beta"] = "assistants=v2"
3261 return headers
3263 @staticmethod
3264 def _assemble_headers(api_key: str | None, request: Request, extra_headers: dict | None = None) -> dict:
3265 base_headers = {}
3266 if api_key is not None:
3267 base_headers = {
3268 "authorization": f"Bearer {api_key}",
3269 "api-key": f"{api_key}",
3270 }
3271 if extra_headers is not None:
3272 base_headers.update(extra_headers)
3273 return BaseOpenAIPassThroughHandler._append_openai_beta_header(
3274 headers=base_headers,
3275 request=request,
3276 )
3279@router.api_route(
3280 "/cursor/{endpoint:path}",
3281 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
3282 tags=["Cursor Pass-through", "pass-through"],
3283)
3284async def cursor_proxy_route(
3285 endpoint: str,
3286 request: Request,
3287 fastapi_response: Response,
3288 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
3289):
3290 """
3291 Pass-through endpoint for the Cursor Cloud Agents API.
3293 Supports all Cursor Cloud Agents endpoints:
3294 - GET /v0/agents — List agents
3295 - POST /v0/agents — Launch an agent
3296 - GET /v0/agents/{id} — Agent status
3297 - GET /v0/agents/{id}/conversation — Agent conversation
3298 - POST /v0/agents/{id}/followup — Add follow-up
3299 - POST /v0/agents/{id}/stop — Stop an agent
3300 - DELETE /v0/agents/{id} — Delete an agent
3301 - GET /v0/me — API key info
3302 - GET /v0/models — List models
3303 - GET /v0/repositories — List GitHub repositories
3305 Uses Basic Authentication (base64-encoded `API_KEY:`).
3307 Credential lookup order:
3308 1. passthrough_endpoint_router (config.yaml deployments with use_in_pass_through)
3309 2. litellm.credential_list (credentials added via UI)
3310 3. CURSOR_API_KEY environment variable
3311 """
3312 import base64
3314 base_target_url = os.getenv("CURSOR_API_BASE") or "https://api.cursor.com"
3316 cursor_api_key = passthrough_endpoint_router.get_credentials(
3317 custom_llm_provider="cursor",
3318 region_name=None,
3319 )
3321 if cursor_api_key is None: 3321 ↛ 3330line 3321 didn't jump to line 3330 because the condition on line 3321 was always true
3322 for credential in litellm.credential_list:
3323 if credential.credential_info and credential.credential_info.get("custom_llm_provider") == "cursor": 3323 ↛ 3324line 3323 didn't jump to line 3324 because the condition on line 3323 was never true
3324 cursor_api_key = credential.credential_values.get("api_key")
3325 credential_api_base = credential.credential_values.get("api_base")
3326 if credential_api_base:
3327 base_target_url = credential_api_base
3328 break
3330 if cursor_api_key is None: 3330 ↛ 3336line 3330 didn't jump to line 3336 because the condition on line 3330 was always true
3331 raise HTTPException(
3332 status_code=401,
3333 detail="Cursor API key not found. Add Cursor credentials via the UI (Models + Endpoints → LLM Credentials) or set CURSOR_API_KEY environment variable.",
3334 )
3336 encoded_endpoint = httpx.URL(endpoint).path
3338 if not encoded_endpoint.startswith("/"):
3339 encoded_endpoint = "/" + encoded_endpoint
3341 base_url: Final = httpx.URL(base_target_url)
3342 updated_url: Final = base_url.copy_with(
3343 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
3344 )
3346 auth_value: Final = base64.b64encode(f"{cursor_api_key}:".encode()).decode("ascii")
3348 endpoint_func: Final = create_pass_through_route(
3349 endpoint=endpoint,
3350 target=str(updated_url),
3351 custom_headers={"Authorization": f"Basic {auth_value}"},
3352 custom_llm_provider="cursor",
3353 )
3354 received_value: Final = await endpoint_func(
3355 request,
3356 fastapi_response,
3357 user_api_key_dict,
3358 )
3360 return received_value
3363TINYFISH_JSON_OBJECT_BODY_DETAIL: Final = (
3364 "TinyFish requests must be a JSON object body sent with Content-Type: application/json."
3365)
3368async def _tinyfish_json_object_field_names(request: Request) -> frozenset[str] | None:
3369 content_type: Final = request.headers.get("content-type", "")
3370 if content_type and not is_json_content_type(content_type):
3371 return None
3372 raw_body: Final = await request.body()
3373 if not raw_body:
3374 return frozenset()
3375 try:
3376 parsed: Final[object] = json.loads(raw_body) # any-ok: json.loads -> Any
3377 except (json.JSONDecodeError, UnicodeDecodeError):
3378 return None
3379 return frozenset(parsed) if isinstance(parsed, dict) else None
3382def _tinyfish_route_timeout() -> float | None:
3383 # only raise the 600s default to cover legal 1200s runs; an operator's configured timeout still wins
3384 proxy_server: Final = sys.modules.get("litellm.proxy.proxy_server")
3385 operator_settings: Final = getattr(proxy_server, "general_settings", None)
3386 operator_timeout: Final = (
3387 operator_settings.get("pass_through_request_timeout") if isinstance(operator_settings, Mapping) else None
3388 )
3389 return None if operator_timeout is not None else TINYFISH_PASSTHROUGH_TIMEOUT_SECONDS
3392@router.api_route(
3393 "/tinyfish/{endpoint:path}",
3394 methods=["GET", "POST"], # mutable-ok: fastapi api_route requires List[str]
3395 tags=["TinyFish Pass-through", "pass-through"], # mutable-ok: fastapi api_route requires a list
3396)
3397async def tinyfish_proxy_route(
3398 endpoint: str,
3399 request: Request,
3400 fastapi_response: Response,
3401 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI dependency injection
3402) -> Response:
3403 """
3404 Pass-through for the TinyFish Agent API (goal-based web automation).
3406 Forwarded endpoints:
3407 - POST /v1/automation/run — run to completion (blocking)
3408 - POST /v1/automation/run-async — submit a run, poll GET /v1/runs/{id} for the result
3409 - POST /v1/automation/run-sse — run with SSE progress events
3410 - GET /v1/runs/{id} — run status / result
3411 - POST /v1/runs/{id}/cancel — cancel a run
3413 Every other Agent API endpoint (vault, wallet, browser profiles, and the GET /v1/runs
3414 listing, which would let any caller discover other callers' run ids) returns 403: all
3415 proxy callers share one upstream key.
3417 Credential lookup order:
3418 1. passthrough_endpoint_router (config.yaml deployments with use_in_pass_through)
3419 2. TINYFISH_API_KEY environment variable
3421 [Docs](https://docs.litellm.ai/docs/pass_through/tinyfish)
3422 """
3423 from .llm_provider_handlers.tinyfish_passthrough_logging_handler import (
3424 resolve_tinyfish_agent_api_base,
3425 )
3427 raw_endpoint_path: Final = httpx.URL(endpoint).path
3428 encoded_endpoint: Final = raw_endpoint_path if raw_endpoint_path.startswith("/") else f"/{raw_endpoint_path}"
3430 if not is_allowed_tinyfish_endpoint(request.method, encoded_endpoint): 3430 ↛ 3438line 3430 didn't jump to line 3438 because the condition on line 3430 was always true
3431 raise HTTPException(
3432 status_code=403,
3433 detail=f"{request.method} {encoded_endpoint} is not an allowed TinyFish Agent passthrough endpoint. "
3434 "Allowed: POST /v1/automation/run, POST /v1/automation/run-async, POST /v1/automation/run-sse, "
3435 "GET /v1/runs/{id}, POST /v1/runs/{id}/cancel.",
3436 )
3438 if request.method == "POST":
3439 body_fields: Final = await _tinyfish_json_object_field_names(request)
3440 if body_fields is None:
3441 raise HTTPException(status_code=400, detail=TINYFISH_JSON_OBJECT_BODY_DETAIL)
3442 envelope_fields: Final = tuple(sorted(body_fields & TINYFISH_REJECTED_ENVELOPE_FIELDS))
3443 if envelope_fields:
3444 raise HTTPException(
3445 status_code=400,
3446 detail=f"Request fields [{', '.join(envelope_fields)}] are LiteLLM pass-through envelope controls "
3447 "and are not accepted on the TinyFish route. Send the native TinyFish request body; streaming is "
3448 "determined by the endpoint.",
3449 )
3450 blocked_fields: Final = tuple(sorted(body_fields & TINYFISH_AUTHENTICATED_RUN_FIELDS))
3451 if (
3452 blocked_fields
3453 and encoded_endpoint.startswith("/v1/automation/")
3454 and str_to_bool(os.getenv("TINYFISH_ALLOW_AUTHENTICATED_RUNS")) is not True
3455 ):
3456 raise HTTPException(
3457 status_code=403,
3458 detail=f"Request fields [{', '.join(blocked_fields)}] run with the shared TinyFish account's saved "
3459 "credentials and are disabled on this proxy. Ask the proxy admin to set "
3460 "TINYFISH_ALLOW_AUTHENTICATED_RUNS=true to allow them.",
3461 )
3463 tinyfish_api_key: Final = passthrough_endpoint_router.get_credentials(
3464 custom_llm_provider="tinyfish",
3465 region_name=None,
3466 )
3467 if tinyfish_api_key is None:
3468 raise HTTPException(
3469 status_code=401,
3470 detail="TinyFish API key not found. Set the TINYFISH_API_KEY environment variable or add a "
3471 "deployment with use_in_pass_through: true.",
3472 )
3474 base_url: Final = httpx.URL(resolve_tinyfish_agent_api_base())
3475 updated_url: Final = base_url.copy_with(
3476 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
3477 )
3479 endpoint_func: Final = create_pass_through_route(
3480 endpoint=endpoint,
3481 target=str(updated_url),
3482 custom_headers=MappingProxyType({"X-API-Key": tinyfish_api_key}),
3483 custom_llm_provider="tinyfish",
3484 timeout=_tinyfish_route_timeout(),
3485 )
3486 received_value: Final = await endpoint_func(
3487 request,
3488 fastapi_response,
3489 user_api_key_dict,
3490 )
3492 return received_value
3495VERTEX_LIVE_UNCONFIGURED_CLOSE_REASON: Final = (
3496 "Vertex AI auth failed: set a use_in_pass_through vertex model, default_vertex_config, or DEFAULT_VERTEXAI_* env"
3497)
3499VERTEX_PUBLISHER_MODEL_PREFIX: Final = "publishers/google/models/"
3501VERTEX_PUBLISHERS_SEGMENT: Final = "publishers/"
3504def _vertex_publisher_model_suffix(model: str) -> str:
3505 """
3506 Turn whatever the client named into the ``publishers/<publisher>/models/<id>`` tail of a Vertex resource name.
3508 Clients send bare ids, LiteLLM ids (``vertex_ai/gemini-live-2.5-flash``), and the Live SDK's ``models/<id>``,
3509 and a publisher model id never contains a slash, so anything ahead of the last one is addressing, not identity
3510 """
3511 publishers_at: Final = model.find(VERTEX_PUBLISHERS_SEGMENT)
3512 if publishers_at != -1:
3513 return model[publishers_at:]
3514 return f"{VERTEX_PUBLISHER_MODEL_PREFIX}{model.rsplit('/', 1)[-1]}"
3517def _get_llm_router() -> Router | None:
3518 from litellm.proxy.proxy_server import llm_router
3520 return llm_router
3523def _resolve_vertex_live_credentials(
3524 vertex_project: str | None,
3525 vertex_location: str | None,
3526 model: str | None,
3527) -> VertexPassThroughCredentials | None:
3528 """
3529 Resolution order: an explicit project/location registration, then ``default_vertex_config`` (which the proxy
3530 fills from the ``DEFAULT_VERTEXAI_*`` env vars whenever the yaml leaves it out), then any DB model entry
3531 flagged ``use_in_pass_through``.
3533 DB entries come last on purpose: an operator who set a global default already said which project
3534 pass-through traffic should bill to, and this route silently ignoring that would be the worse surprise
3535 """
3536 keyed: Final = passthrough_endpoint_router.get_vertex_credentials(
3537 project_id=vertex_project,
3538 location=vertex_location,
3539 )
3540 if keyed is not None and keyed.vertex_project is not None:
3541 return keyed
3542 from_deployments: Final = passthrough_endpoint_router.get_vertex_credentials_from_router_deployments(model=model)
3543 if from_deployments is not None:
3544 return from_deployments
3545 if keyed is not None:
3546 return keyed
3547 passthrough_endpoint_router.set_default_vertex_config()
3548 return passthrough_endpoint_router.get_vertex_credentials(
3549 project_id=vertex_project,
3550 location=vertex_location,
3551 )
3554def _build_vertex_live_setup_model_rewriter(
3555 vertex_project: str | None,
3556 vertex_location: str | None,
3557 llm_router: Router | None,
3558) -> Callable[[str], str] | None:
3559 """
3560 Rewrite the ``setup`` frame's model into the full Vertex resource path the Live API requires.
3562 Clients address the gateway the way they address LiteLLM (bare id or model alias); Vertex reads anything
3563 that is not a ``projects/...`` path as a project name and closes the socket
3564 """
3565 if vertex_project is None or vertex_location is None:
3566 return None
3568 def rewrite(setup_model: str) -> str:
3569 if setup_model.startswith("projects/"):
3570 return setup_model
3571 aliased: Final = _resolve_alias_to_upstream_model(setup_model, llm_router)
3572 return f"projects/{vertex_project}/locations/{vertex_location}/{_vertex_publisher_model_suffix(aliased)}"
3574 return rewrite
3577def _resolve_alias_to_upstream_model(setup_model: str, llm_router: Router | None) -> str:
3578 """
3579 The Live SDK wraps whatever the caller typed as ``models/<name>``, so a gateway alias arrives prefixed
3580 """
3581 if llm_router is None:
3582 return setup_model
3583 candidates: Final = (setup_model, setup_model.rsplit("/", 1)[-1])
3584 upstream: Final = next(
3585 (
3586 deployment["litellm_params"].get("model")
3587 for deployment in (llm_router.get_model_list() or ())
3588 if deployment.get("model_name") in candidates
3589 ),
3590 None,
3591 )
3592 if upstream is None:
3593 return setup_model
3594 try:
3595 _, provider, _, _ = litellm.get_llm_provider(model=upstream)
3596 except litellm.exceptions.BadRequestError:
3597 return upstream
3598 return upstream.removeprefix(f"{provider}/")
3601async def vertex_ai_live_websocket_passthrough(
3602 websocket: WebSocket,
3603 model: str | None = None,
3604 vertex_project: str | None = None,
3605 vertex_location: str | None = None,
3606 user_api_key_dict: UserAPIKeyAuth | None = None,
3607):
3608 """
3609 Vertex AI Live API WebSocket Pass-through Function
3611 This function provides WebSocket passthrough functionality for Vertex AI Live API,
3612 allowing real-time communication with Google's Live API service.
3614 Note: This function should be registered in proxy_server.py using:
3615 app.websocket("/vertex_ai/live")(vertex_ai_live_websocket_passthrough)
3616 """
3617 from litellm.proxy.proxy_server import proxy_logging_obj
3619 _ = user_api_key_dict # passthrough route already authenticated; avoid lint warnings
3621 await websocket.accept()
3623 incoming_headers: Final = dict(websocket.headers)
3624 vertex_credentials_config: Final = _resolve_vertex_live_credentials(
3625 vertex_project=vertex_project,
3626 vertex_location=vertex_location,
3627 model=model,
3628 )
3630 configured_project: Final = vertex_project or (
3631 vertex_credentials_config.vertex_project if vertex_credentials_config is not None else None
3632 )
3633 configured_location: Final = vertex_location or (
3634 vertex_credentials_config.vertex_location if vertex_credentials_config is not None else None
3635 )
3636 credentials_value: Final = (
3637 vertex_credentials_config.vertex_credentials if vertex_credentials_config is not None else None
3638 )
3640 try:
3641 resolved_location: Final = (
3642 vertex_llm_base.get_vertex_region(
3643 vertex_region=configured_location or vertex_llm_base.get_default_vertex_location(),
3644 model=model,
3645 )
3646 if model
3647 else configured_location or vertex_llm_base.get_default_vertex_location()
3648 )
3650 (
3651 access_token,
3652 resolved_project,
3653 ) = await vertex_llm_base._ensure_access_token_async(
3654 credentials=credentials_value,
3655 project_id=configured_project,
3656 custom_llm_provider="vertex_ai_beta",
3657 )
3658 except Exception as e:
3659 verbose_proxy_logger.exception("Failed to prepare Vertex AI credentials for live passthrough")
3660 # Log the authentication failure using proxy_logging_obj
3661 if proxy_logging_obj and user_api_key_dict:
3662 await proxy_logging_obj.post_call_failure_hook(
3663 user_api_key_dict=user_api_key_dict,
3664 original_exception=e,
3665 request_data={},
3666 )
3667 if websocket.client_state != WebSocketState.DISCONNECTED:
3668 await websocket.close(code=1011, reason=VERTEX_LIVE_UNCONFIGURED_CLOSE_REASON)
3669 return
3671 host_location: Final = resolved_location or vertex_llm_base.get_default_vertex_location()
3672 host: Final = get_vertex_base_url(host_location).removeprefix("https://").rstrip("/")
3673 service_url: Final = f"wss://{host}/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent"
3675 upstream_headers: Final = {
3676 "Authorization": f"Bearer {access_token}",
3677 "Content-Type": "application/json",
3678 }
3679 if resolved_project:
3680 upstream_headers["x-goog-user-project"] = resolved_project
3682 # Forward any custom x-goog-* headers provided by the caller if we haven't overridden them
3683 for header_name, header_value in incoming_headers.items():
3684 lower_header = header_name.lower()
3685 if lower_header.startswith("x-goog-") and header_name not in upstream_headers:
3686 upstream_headers[header_name] = header_value
3688 # Use the new WebSocket passthrough pattern
3689 if user_api_key_dict is None:
3690 raise ValueError("user_api_key_dict is required for WebSocket passthrough")
3692 return await websocket_passthrough_request(
3693 websocket=websocket,
3694 target=service_url,
3695 custom_headers=upstream_headers,
3696 user_api_key_dict=user_api_key_dict,
3697 forward_headers=False,
3698 endpoint="/vertex_ai/live",
3699 accept_websocket=False,
3700 setup_model_rewriter=_build_vertex_live_setup_model_rewriter(
3701 vertex_project=resolved_project,
3702 vertex_location=resolved_location,
3703 llm_router=_get_llm_router(),
3704 ),
3705 )
3708def create_vertex_ai_live_websocket_endpoint():
3709 """
3710 Create a Vertex AI Live WebSocket endpoint using the new passthrough pattern.
3712 This demonstrates how to use the create_websocket_passthrough_route function
3713 for a provider-specific WebSocket endpoint.
3714 """
3715 # This would be used like:
3716 # endpoint_func = create_vertex_ai_live_websocket_endpoint()
3717 # app.websocket("/vertex_ai/live")(endpoint_func)
3719 # For now, we'll keep the existing implementation since it has
3720 # provider-specific logic for Vertex AI credentials and headers
3721 return vertex_ai_live_websocket_passthrough
3724def create_generic_websocket_passthrough_endpoint(
3725 provider: str,
3726 target_url: str,
3727 custom_headers: dict | None = None,
3728 forward_headers: bool = False,
3729 cost_per_request: float | None = None,
3730):
3731 """
3732 Create a generic WebSocket passthrough endpoint for any provider.
3734 This demonstrates the new WebSocket passthrough pattern that's similar to
3735 the HTTP create_pass_through_route function.
3737 Args:
3738 provider: The provider name (e.g., "anthropic", "cohere")
3739 target_url: The target WebSocket URL
3740 custom_headers: Custom headers to include
3741 forward_headers: Whether to forward incoming headers
3743 Returns:
3744 A WebSocket endpoint function that can be registered with app.websocket()
3746 Example usage:
3747 # Create a WebSocket endpoint for Anthropic
3748 anthropic_ws_func = create_generic_websocket_passthrough_endpoint(
3749 provider="anthropic",
3750 target_url="wss://api.anthropic.com/v1/ws",
3751 custom_headers={"x-api-key": "your-api-key"},
3752 forward_headers=True
3753 )
3755 # Register it in proxy_server.py
3756 app.websocket("/anthropic/ws")(anthropic_ws_func)
3757 """
3758 return create_websocket_passthrough_route(
3759 endpoint=f"/{provider}/ws",
3760 target=target_url,
3761 custom_headers=custom_headers,
3762 _forward_headers=forward_headers,
3763 cost_per_request=cost_per_request,
3764 )
3767@router.api_route(
3768 "/gigachat/{endpoint:path}",
3769 methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route methods
3770 tags=["Gigachat Pass-through", "pass-through"], # mutable-ok: FastAPI route tags
3771)
3772async def gigachat_proxy_route(
3773 endpoint: str,
3774 request: Request,
3775 fastapi_response: Response,
3776 user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
3777) -> Response:
3778 """
3779 [Docs](https://docs.litellm.ai/docs/pass_through/gigachat)
3780 """
3781 from litellm.proxy.proxy_server import (
3782 general_settings,
3783 llm_router,
3784 proxy_config,
3785 proxy_logging_obj,
3786 select_data_generator,
3787 user_api_base,
3788 user_max_tokens,
3789 user_model,
3790 user_request_timeout,
3791 user_temperature,
3792 version,
3793 )
3795 ## check for streaming
3796 request_body: Final[dict[str, object]] = await get_request_body(request)
3797 is_router_model = False # rebind-ok: conditionally set to True when model uses router
3799 raw_model: Final = request_body.get("model")
3800 model: Final = raw_model if isinstance(raw_model, str) else None
3801 if model: 3801 ↛ 3802line 3801 didn't jump to line 3802 because the condition on line 3801 was never true
3802 is_router_model = is_passthrough_request_using_router_model(request_body, llm_router)
3803 elif any(word in endpoint for word in ("completions", "embeddings")): 3803 ↛ 3804line 3803 didn't jump to line 3804 because the condition on line 3803 was never true
3804 raise HTTPException(status_code=400, detail={"error": "Model is required in request body"})
3806 # If router model, use dedicated router passthrough handler
3807 # This uses the same common processing path as non-router models
3808 if model and is_router_model and llm_router: 3808 ↛ 3809line 3808 didn't jump to line 3809 because the condition on line 3808 was never true
3809 return await handle_gigachat_passthrough_router_model(
3810 model=model,
3811 endpoint=endpoint,
3812 request=request,
3813 request_body=request_body,
3814 fastapi_response=fastapi_response,
3815 llm_router=llm_router,
3816 user_api_key_dict=user_api_key_dict,
3817 proxy_logging_obj=proxy_logging_obj,
3818 general_settings=general_settings,
3819 proxy_config=proxy_config,
3820 select_data_generator=select_data_generator,
3821 user_model=user_model,
3822 user_temperature=user_temperature,
3823 user_request_timeout=user_request_timeout,
3824 user_max_tokens=user_max_tokens,
3825 user_api_base=user_api_base,
3826 version=version,
3827 )
3829 verbose_proxy_logger.debug(
3830 "Gigachat passthrough: Using direct Gigachat model '%s' for endpoint '%s'", model, endpoint
3831 )
3833 from litellm.llms.gigachat.authenticator import get_access_token
3834 from litellm.llms.gigachat.utils import GIGACHAT_BASE_URL
3836 base_target_url: Final = get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL
3837 request_path: Final = httpx.URL(endpoint).path
3838 encoded_endpoint: Final = request_path if request_path.startswith("/") else f"/{request_path}"
3840 base_url: Final = httpx.URL(base_target_url)
3841 updated_url: Final = base_url.copy_with(
3842 path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
3843 )
3845 is_streaming_request: Final = await is_streaming_request_fn(request)
3847 endpoint_func: Final = create_pass_through_route(
3848 endpoint=endpoint,
3849 target=str(updated_url),
3850 custom_headers={"Authorization": f"Bearer {get_access_token()}"},
3851 is_streaming_request=is_streaming_request,
3852 )
3853 return await endpoint_func(
3854 request,
3855 fastapi_response,
3856 user_api_key_dict,
3857 )
3860async def handle_gigachat_passthrough_router_model(
3861 model: str,
3862 endpoint: str,
3863 request: Request,
3864 request_body: dict,
3865 fastapi_response: Response,
3866 llm_router: litellm.Router,
3867 user_api_key_dict: UserAPIKeyAuth,
3868 proxy_logging_obj: ProxyLoggingType,
3869 general_settings: dict,
3870 proxy_config: ProxyConfig,
3871 select_data_generator: Callable,
3872 user_model: str | None,
3873 user_temperature: float | None,
3874 user_request_timeout: float | None,
3875 user_max_tokens: int | None,
3876 user_api_base: str | None,
3877 version: str | None,
3878) -> Response | StreamingResponse:
3879 """
3880 Handle Gigachat passthrough for router models (models defined in config.yaml).
3882 Uses the same common processing path as non-router models to ensure
3883 metadata and hooks are properly initialized.
3885 Args:
3886 model: The router model name (e.g., "gigachat/gigachat-2")
3887 endpoint: The Gigachat endpoint path (e.g., "/chat/completions")
3888 request: The FastAPI request object
3889 request_body: The parsed request body
3890 llm_router: The LiteLLM router instance
3891 user_api_key_dict: The user API key authentication dictionary
3892 proxy_logging_obj: Proxy logging
3893 general_settings: Proxy general settings
3894 proxy_config: Proxy config
3895 select_data_generator: Select data generator function
3896 (additional args for common processing)
3898 Returns:
3899 Response or StreamingResponse depending on endpoint type
3900 """
3901 from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
3903 is_streaming: Final = request_body.get("stream", False) # pyright: ignore[reportUnknownVariableType] # request_body is dict[Unknown, Unknown]
3905 data: Final[dict[str, object]] = await _read_request_body(request=request)
3906 if user_api_key_dict is not None:
3907 auth_metadata: Final = {
3908 metadata_key: value
3909 for metadata_key, value in (
3910 ("user_api_key_user_id", getattr(user_api_key_dict, "user_id", None)),
3911 ("user_api_key_team_id", getattr(user_api_key_dict, "team_id", None)),
3912 ("user_api_key_org_id", getattr(user_api_key_dict, "org_id", None)),
3913 ("agent_id", getattr(user_api_key_dict, "agent_id", None)),
3914 )
3915 if value is not None
3916 }
3917 existing_metadata: Final = data.get("metadata")
3918 data["metadata"] = {
3919 **(existing_metadata if isinstance(existing_metadata, dict) else {}),
3920 **auth_metadata,
3921 }
3923 verbose_proxy_logger.debug(
3924 "Gigachat router passthrough: model='%s', endpoint='%s', streaming=%s", model, endpoint, is_streaming
3925 )
3927 # Use the common processing path (same as non-router models)
3928 # This ensures all metadata, hooks, and logging are properly initialized
3930 data["model"] = model
3931 data["method"] = request.method
3932 data["endpoint"] = endpoint
3933 data["json"] = request_body
3934 data["custom_llm_provider"] = "gigachat"
3936 keys: Final = [ # mutable-ok: list of keys to remove from data
3937 "gigachat_auth_url",
3938 "gigachat_access_token",
3939 "gigachat_scope",
3940 "api_base",
3941 "api_key",
3942 ]
3943 for key in keys:
3944 data.pop(key, None)
3946 client: Final = get_async_httpx_client(
3947 llm_provider=LlmProviders.GIGACHAT,
3948 params={ # mutable-ok: httpx client params
3949 "timeout": httpx.Timeout(timeout=600.0, connect=5.0),
3950 },
3951 )
3953 data["client"] = client
3954 base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
3956 # Use the common passthrough processing to handle metadata and hooks
3957 # This also handles all response formatting (streaming/non-streaming) and exceptions
3958 try:
3959 result = await base_llm_response_processor.base_passthrough_process_llm_request( # rebind-ok: assigned once in try block
3960 request=request,
3961 fastapi_response=fastapi_response,
3962 user_api_key_dict=user_api_key_dict,
3963 proxy_logging_obj=proxy_logging_obj,
3964 llm_router=llm_router,
3965 general_settings=general_settings,
3966 proxy_config=proxy_config,
3967 select_data_generator=select_data_generator,
3968 model=model,
3969 user_model=user_model,
3970 user_temperature=user_temperature,
3971 user_request_timeout=user_request_timeout,
3972 user_max_tokens=user_max_tokens,
3973 user_api_base=user_api_base,
3974 version=version,
3975 )
3976 except Exception as e: # noqa: BLE001 # Safe catch-all for handle exception
3977 # Use common exception handling
3978 raise await base_llm_response_processor._handle_llm_api_exception(
3979 e=e,
3980 user_api_key_dict=user_api_key_dict,
3981 proxy_logging_obj=proxy_logging_obj,
3982 )
3983 else:
3984 if isinstance(result, StreamingResponse):
3985 if result.headers.get("Content-Type") is None:
3986 result.headers["Content-Type"] = "text/event-stream; charset=utf-8"
3988 return result
3991@router.api_route(
3992 "/watsonx/{endpoint:path}",
3993 methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
3994 tags=["Watsonx Pass-through", "pass-through"],
3995)
3996async def watsonx_proxy_route(
3997 endpoint: str,
3998 request: Request,
3999 fastapi_response: Response,
4000 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
4001):
4002 """
4003 Watsonx pass-through endpoint.
4004 Allows using Watsonx APIs with automatic IAM token management and version parameter injection.
4006 Example:
4007 POST /watsonx/ml/v1/text/tokenization
4008 POST /watsonx/ml/v1/text/generation
4009 """
4010 # Direct passthrough with WatsonxPassthroughConfig
4011 from litellm.types.utils import LlmProviders
4012 from litellm.utils import ProviderConfigManager
4014 provider_config: Final = ProviderConfigManager.get_provider_passthrough_config(
4015 provider=LlmProviders.WATSONX,
4016 model="",
4017 )
4019 if provider_config is None: 4019 ↛ 4020line 4019 didn't jump to line 4020 because the condition on line 4019 was never true
4020 raise HTTPException(status_code=404, detail="Watsonx passthrough config not found")
4022 # Get complete URL with version parameter
4023 complete_url, _ = provider_config.get_complete_url(
4024 api_base=None,
4025 api_key=None,
4026 model="",
4027 endpoint=endpoint,
4028 request_query_params=None,
4029 litellm_params={},
4030 )
4032 # Get auth headers with IAM token
4033 auth_headers: Final = provider_config.validate_environment(
4034 headers={},
4035 model="",
4036 messages=[],
4037 optional_params={},
4038 litellm_params={},
4039 api_key=None,
4040 api_base=None,
4041 )
4043 # Check for streaming
4044 is_streaming_request = False
4045 if request.method == "POST":
4046 if "multipart/form-data" not in request.headers.get("content-type", ""):
4047 _request_body = await _json_request_body(request)
4048 else:
4049 _request_body = await get_form_data(request)
4051 if _request_body.get("stream"):
4052 is_streaming_request = True
4054 request_query_params: Final = dict(request.query_params)
4055 if request_query_params.get("version") is None:
4056 request_query_params["version"] = litellm.WATSONX_DEFAULT_API_VERSION
4058 # Create pass-through endpoint
4059 endpoint_func: Final = create_pass_through_route(
4060 endpoint=endpoint,
4061 target=str(complete_url),
4062 custom_headers=auth_headers,
4063 is_streaming_request=is_streaming_request,
4064 custom_llm_provider="watsonx",
4065 query_params=request_query_params,
4066 )
4068 return await endpoint_func(
4069 request,
4070 fastapi_response,
4071 user_api_key_dict,
4072 )