Coverage for .venv/lib/python3.13/site-packages/litellm/proxy/route_llm_request.py: 54%
203 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-10 12:01 +0000
1import asyncio
2from collections.abc import Mapping
3from typing import TYPE_CHECKING, Any, Final, Literal
5import httpx
6from fastapi import HTTPException, status
8import litellm
9from litellm.proxy._types import ProxyException, UserAPIKeyAuth
10from litellm.router_utils.common_utils import _is_proxy_admin_request
12# Client-supplied params that make the router or the call path fabricate a
13# failure or a delay instead of calling the provider. The ``mock_testing_*``
14# names are kept in sync with ``litellm.types.router.MockRouterTestingParams``
15# by ``test_gated_mock_params_cover_mock_router_testing_params``. Hardcoding
16# (rather than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at
17# import time) avoids a cyclic import: ``litellm.types.router`` imports
18# back into proxy modules before this module finishes loading.
19GATED_MOCK_PARAM_NAMES: Final[tuple[str, ...]] = (
20 "mock_testing_fallbacks",
21 "mock_testing_context_fallbacks",
22 "mock_testing_content_policy_fallbacks",
23 "mock_testing_rate_limit_error",
24 "mock_timeout",
25 "mock_delay",
26)
28MOCK_TESTING_CONFIG_KEY: Final = "dangerously_allow_mock_testing_request_params"
30if TYPE_CHECKING: 30 ↛ 31line 30 didn't jump to line 31 because the condition on line 30 was never true
31 from litellm.router import Router as _Router
33 LitellmRouter = _Router
34else:
35 LitellmRouter = Any
38def _route_user_config_request(data: dict, route_type: str):
39 """Route a request using the user-provided router config."""
40 router_config: Final = data.pop("user_config")
42 # Filter router_config to only include valid Router.__init__ arguments
43 # This prevents TypeError when invalid parameters are stored in the database
44 valid_args: Final = litellm.Router.get_valid_args()
45 filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args}
47 user_router: Final = litellm.Router(**filtered_config)
48 ret_val: Final = getattr(user_router, f"{route_type}")(**data)
49 user_router.discard()
50 return ret_val
53def _is_a2a_agent_model(model_name: object) -> bool:
54 """Check if the model name is for an A2A agent (a2a/ prefix)."""
55 return isinstance(model_name, str) and model_name.startswith("a2a/")
58def _raise_if_model_fully_blocked(llm_router: LitellmRouter, model_name: object, team_id: str | None) -> None:
59 if not isinstance(model_name, str) or not model_name:
60 return
61 if not isinstance(llm_router, litellm.Router): 61 ↛ 62line 61 didn't jump to line 62 because the condition on line 61 was never true
62 return
63 deployments: Final = llm_router.get_model_list(model_name=model_name, team_id=team_id) or []
64 if llm_router._are_all_deployments_blocked(deployments): 64 ↛ 65line 64 didn't jump to line 65 because the condition on line 64 was never true
65 raise litellm.PermissionDeniedError(
66 message="Model is blocked",
67 model=model_name,
68 llm_provider="",
69 response=httpx.Response(
70 status_code=403,
71 request=httpx.Request(method="POST", url="https://github.com/BerriAI/litellm"),
72 ),
73 )
76ROUTE_ENDPOINT_MAPPING: Final = {
77 "acompletion": "/chat/completions",
78 "atext_completion": "/completions",
79 "aembedding": "/embeddings",
80 "aimage_generation": "/image/generations",
81 "aspeech": "/audio/speech",
82 "atranscription": "/audio/transcriptions",
83 "amoderation": "/moderations",
84 "arerank": "/rerank",
85 "aresponses": "/responses",
86 "_aresponses_websocket": "/responses",
87 "alist_input_items": "/responses/{response_id}/input_items",
88 "aimage_edit": "/images/edits",
89 "acancel_responses": "/responses/{response_id}/cancel",
90 "acompact_responses": "/responses/compact",
91 "aocr": "/ocr",
92 "asearch": "/search",
93 "adecisions": "/decisions",
94 "avideo_generation": "/videos",
95 "avideo_list": "/videos",
96 "avideo_status": "/videos/{video_id}",
97 "avideo_content": "/videos/{video_id}/content",
98 "avideo_remix": "/videos/{video_id}/remix",
99 "avideo_create_character": "/videos/characters",
100 "avideo_get_character": "/videos/characters/{character_id}",
101 "avideo_edit": "/videos/edits",
102 "avideo_extension": "/videos/extensions",
103 "acreate_realtime_client_secret": "/realtime/client_secrets",
104 "arealtime_calls": "/realtime/calls",
105 "acreate_realtime_transcription_session": "/realtime/transcription_sessions",
106 "acreate_container": "/containers",
107 "alist_containers": "/containers",
108 "aretrieve_container": "/containers/{container_id}",
109 "adelete_container": "/containers/{container_id}",
110 # Auto-generated container file routes
111 "aupload_container_file": "/containers/{container_id}/files",
112 "alist_container_files": "/containers/{container_id}/files",
113 "aretrieve_container_file": "/containers/{container_id}/files/{file_id}",
114 "adelete_container_file": "/containers/{container_id}/files/{file_id}",
115 "aretrieve_container_file_content": "/containers/{container_id}/files/{file_id}/content",
116 "acreate_skill": "/skills",
117 "alist_skills": "/skills",
118 "aget_skill": "/skills/{skill_id}",
119 "adelete_skill": "/skills/{skill_id}",
120 "aingest": "/rag/ingest",
121 # Google Interactions API routes
122 "acreate_interaction": "/interactions",
123 "aget_interaction": "/interactions/{interaction_id}",
124 "adelete_interaction": "/interactions/{interaction_id}",
125 "acancel_interaction": "/interactions/{interaction_id}/cancel",
126 # Google Managed Agents API routes
127 "acreate_agent": "/v1beta/agents",
128 "alist_agents": "/v1beta/agents",
129 "aget_agent": "/v1beta/agents/{name}",
130 "adelete_agent": "/v1beta/agents/{name}",
131 "alist_agent_versions": "/v1beta/agents/{name}/versions",
132 # OpenAI Evals API routes
133 "acreate_eval": "/evals",
134 "alist_evals": "/evals",
135 "aget_eval": "/evals/{eval_id}",
136 "aupdate_eval": "/evals/{eval_id}",
137 "adelete_eval": "/evals/{eval_id}",
138 "acancel_eval": "/evals/{eval_id}/cancel",
139 # OpenAI Evals Runs API routes
140 "acreate_run": "/evals/{eval_id}/runs",
141 "alist_runs": "/evals/{eval_id}/runs",
142 "aget_run": "/evals/{eval_id}/runs/{run_id}",
143 "acancel_run": "/evals/{eval_id}/runs/{run_id}/cancel",
144 "adelete_run": "/evals/{eval_id}/runs/{run_id}",
145 "acreate_batch": "/batches",
146}
149_AVAILABLE_MODELS_HINT: Final = "Call `/v1/models` to view available models for your key."
152class ProxyModelNotFoundError(HTTPException):
153 def __init__(self, route: str, model_name: str, retryable_with_model_read_through: bool = True):
154 self.retryable_with_model_read_through: Final = retryable_with_model_read_through
155 self.spend_log_error_message: Final = f"{route}: Invalid model name passed in. {_AVAILABLE_MODELS_HINT}"
156 detail: Final = {"error": f"{route}: Invalid model name passed in model={model_name}. {_AVAILABLE_MODELS_HINT}"}
157 super().__init__(status_code=status.HTTP_400_BAD_REQUEST, detail=detail)
160REQUIRED_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, ...]]] = {
161 "acompletion": ("messages",),
162 "aembedding": ("input",),
163 "aresponses": ("input",),
164 "acreate_batch": ("input_file_id", "endpoint", "completion_window"),
165}
168class ProxyMissingRequiredParamError(ProxyException):
169 def __init__(self, route: str, param: str):
170 super().__init__(
171 message=f"{route}: Missing required parameter: '{param}'.",
172 type="invalid_request_error",
173 param=param,
174 code=status.HTTP_400_BAD_REQUEST,
175 )
178def raise_if_required_body_param_missing(route_type: str, data: Mapping[str, object]) -> None:
179 missing_param: Final = next(
180 (param for param in REQUIRED_BODY_PARAMS_BY_ROUTE.get(route_type, ()) if data.get(param) is None),
181 None,
182 )
183 if missing_param is None:
184 return
185 raise ProxyMissingRequiredParamError(
186 route=ROUTE_ENDPOINT_MAPPING.get(route_type, route_type),
187 param=missing_param,
188 )
191class MockTestingParamsDisabledError(HTTPException):
192 def __init__(self, params: tuple[str, ...]):
193 super().__init__(
194 status_code=status.HTTP_400_BAD_REQUEST,
195 detail={ # mutable-ok: HTTPException.detail has no immutable form; same shape as the sibling errors here
196 "error": (
197 f"Mock testing request params are disabled on this proxy: {', '.join(params)}. "
198 f"An admin can enable them by setting `general_settings.{MOCK_TESTING_CONFIG_KEY}: true` "
199 "in config.yaml. This setting cannot be changed from the Admin UI or the API."
200 )
201 },
202 )
205def raise_if_mock_testing_params_disallowed(data: Mapping[str, object], *, allowed: bool) -> None:
206 """Reject client-supplied mock testing params unless an admin opted in.
208 Rejecting (rather than silently dropping) keeps a request that asked for a
209 synthetic failure from returning a normal success, which reads as a passing
210 fallback test that never ran.
211 """
212 if allowed: 212 ↛ 213line 212 didn't jump to line 213 because the condition on line 212 was never true
213 return
214 present: Final = tuple(name for name in GATED_MOCK_PARAM_NAMES if name in data)
215 if present: 215 ↛ 216line 215 didn't jump to line 216 because the condition on line 215 was never true
216 raise MockTestingParamsDisabledError(params=present)
219def mock_testing_params_allowed() -> bool:
220 """Read the opt-in from the running proxy's ``general_settings``."""
221 from litellm.proxy import proxy_server
223 return proxy_server.general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is True
226def get_team_id_from_data(data: dict) -> str | None:
227 """
228 Get the team id from the data's metadata or litellm_metadata params.
229 """
230 if "metadata" in data and data["metadata"] is not None and "user_api_key_team_id" in data["metadata"]:
231 return data["metadata"].get("user_api_key_team_id")
232 elif ( 232 ↛ 238line 232 didn't jump to line 238 because the condition on line 232 was always true
233 "litellm_metadata" in data
234 and data["litellm_metadata"] is not None
235 and "user_api_key_team_id" in data["litellm_metadata"]
236 ):
237 return data["litellm_metadata"].get("user_api_key_team_id")
238 return None
241_shared_session_lock: asyncio.Lock | None = None
244def _get_shared_session_lock() -> asyncio.Lock:
245 """Lazily create the shared session lock (must be called within a running event loop).
247 WARNING: Do not reset _shared_session_lock to None while any coroutine may be
248 executing the session-recovery path; doing so breaks the double-checked locking
249 guarantee and can cause duplicate session creation.
250 """
251 global _shared_session_lock
252 if _shared_session_lock is None:
253 _shared_session_lock = asyncio.Lock()
254 return _shared_session_lock
257async def add_shared_session_to_data(data: dict) -> None:
258 """
259 Add shared aiohttp session for connection reuse (prevents cold starts).
260 If the session was closed (e.g. due to network interruption or idle timeout),
261 automatically recreates it so connection pooling is restored.
262 Uses an asyncio.Lock to prevent race conditions where multiple concurrent
263 requests could each create a new session, leaking intermediate ones.
264 Silently continues without session reuse if import fails or session is unavailable.
266 Args:
267 data: Dictionary to add the shared session to
268 """
269 try:
270 from litellm._logging import verbose_proxy_logger
271 from litellm.proxy import proxy_server
273 session = proxy_server.shared_aiohttp_session
275 if session is not None and not session.closed: 275 ↛ 278line 275 didn't jump to line 278 because the condition on line 275 was always true
276 data["shared_session"] = session
277 verbose_proxy_logger.info("SESSION REUSE: Attached shared aiohttp session to request (ID: %s)", id(session))
278 elif session is not None and session.closed:
279 # Session was created at startup but has since closed — recreate it
280 # Use lock to prevent concurrent recreation (avoids session/connector leak)
281 lock: Final = _get_shared_session_lock()
282 async with lock:
283 # Double-check under lock — another coroutine may have already recreated it
284 session = proxy_server.shared_aiohttp_session
285 if session is not None and not session.closed:
286 data["shared_session"] = session
287 return
289 # session could be None here (if another coroutine set it to None)
290 # or closed — either way we need to recreate
291 if session is not None:
292 verbose_proxy_logger.warning(
293 "SESSION REUSE: Shared aiohttp session is closed (ID: %s), recreating...", id(session)
294 )
295 else:
296 verbose_proxy_logger.warning(
297 "SESSION REUSE: Shared aiohttp session is None after re-check, recreating..."
298 )
299 try:
300 new_session = await proxy_server._initialize_shared_aiohttp_session()
301 except Exception:
302 verbose_proxy_logger.exception("SESSION REUSE: Exception during shared session recreation")
303 new_session = None
304 if new_session is not None:
305 proxy_server.shared_aiohttp_session = new_session
306 data["shared_session"] = new_session
307 else:
308 verbose_proxy_logger.info(
309 "SESSION REUSE: Failed to recreate shared session, continuing without session reuse"
310 )
311 else:
312 verbose_proxy_logger.info("SESSION REUSE: No shared session available for this request")
313 except Exception:
314 # Continue without session reuse — this outer handler covers import failures
315 # and other unexpected errors to avoid breaking the request path.
316 # Inner recovery logic has its own specific exception handling.
317 try:
318 from litellm._logging import verbose_proxy_logger
320 verbose_proxy_logger.debug(
321 "SESSION REUSE: Unexpected error in session setup, continuing without reuse",
322 exc_info=True,
323 )
324 except Exception:
325 pass
328RouteType = Literal[
329 "acompletion",
330 "atext_completion",
331 "aembedding",
332 "aimage_generation",
333 "aspeech",
334 "atranscription",
335 "amoderation",
336 "arerank",
337 "aresponses",
338 "aget_responses",
339 "adelete_responses",
340 "acancel_responses",
341 "acompact_responses",
342 "acreate_response_reply",
343 "alist_input_items",
344 "_arealtime", # private function for realtime API
345 "acreate_realtime_client_secret",
346 "arealtime_calls",
347 "acreate_realtime_transcription_session",
348 "_aresponses_websocket", # private function for responses WebSocket mode
349 "aimage_edit",
350 "agenerate_content",
351 "agenerate_content_stream",
352 "allm_passthrough_route",
353 "acreate_batch",
354 "aretrieve_batch",
355 "alist_batches",
356 "afile_content",
357 "afile_retrieve",
358 "acreate_fine_tuning_job",
359 "acancel_fine_tuning_job",
360 "alist_fine_tuning_jobs",
361 "aretrieve_fine_tuning_job",
362 "avector_store_search",
363 "avector_store_create",
364 "avector_store_retrieve",
365 "avector_store_list",
366 "avector_store_update",
367 "avector_store_delete",
368 "avector_store_file_create",
369 "avector_store_file_list",
370 "avector_store_file_retrieve",
371 "avector_store_file_content",
372 "avector_store_file_update",
373 "avector_store_file_delete",
374 "aocr",
375 "asearch",
376 "adecisions",
377 "avideo_generation",
378 "avideo_list",
379 "avideo_status",
380 "avideo_content",
381 "avideo_remix",
382 "avideo_create_character",
383 "avideo_get_character",
384 "avideo_edit",
385 "avideo_extension",
386 "acreate_container",
387 "alist_containers",
388 "aretrieve_container",
389 "adelete_container",
390 "aupload_container_file",
391 "alist_container_files",
392 "aretrieve_container_file",
393 "adelete_container_file",
394 "aretrieve_container_file_content",
395 "acreate_skill",
396 "alist_skills",
397 "aget_skill",
398 "adelete_skill",
399 "aingest",
400 "anthropic_messages",
401 "acreate_interaction",
402 "aget_interaction",
403 "adelete_interaction",
404 "acancel_interaction",
405 "acreate_agent",
406 "alist_agents",
407 "aget_agent",
408 "adelete_agent",
409 "alist_agent_versions",
410 "asend_message",
411 "call_mcp_tool",
412 "acancel_batch",
413 "afile_delete",
414 "acreate_eval",
415 "alist_evals",
416 "aget_eval",
417 "aupdate_eval",
418 "adelete_eval",
419 "acancel_eval",
420 "acreate_run",
421 "alist_runs",
422 "aget_run",
423 "acancel_run",
424 "adelete_run",
425]
428async def route_request(
429 data: dict,
430 llm_router: LitellmRouter | None,
431 user_model: str | None,
432 route_type: RouteType,
433 user_api_key_dict: UserAPIKeyAuth | None = None,
434):
435 """
436 Common helper to route the request
437 """
438 try:
439 return await _route_request_single_attempt(
440 data=data,
441 llm_router=llm_router,
442 user_model=user_model,
443 route_type=route_type,
444 user_api_key_dict=user_api_key_dict,
445 )
446 except ProxyModelNotFoundError as e:
447 requested_model: Final = data.get("model", "")
448 if not e.retryable_with_model_read_through or not isinstance(requested_model, str) or not requested_model:
449 raise
450 from litellm.proxy import proxy_server
451 from litellm.proxy.common_utils.registry_read_through import (
452 model_registry_read_through,
453 )
455 if not await model_registry_read_through.attempt(requested_model): 455 ↛ 457line 455 didn't jump to line 457 because the condition on line 455 was always true
456 raise
457 return await _route_request_single_attempt(
458 data=data,
459 llm_router=proxy_server.llm_router,
460 user_model=user_model,
461 route_type=route_type,
462 user_api_key_dict=user_api_key_dict,
463 )
466async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited provider coroutines; the inferred union keeps route_request's callers typed
467 data: dict, # mutable-ok: request body is the proxy-wide mutable dict contract shared with route_request
468 llm_router: LitellmRouter | None,
469 user_model: str | None,
470 route_type: RouteType,
471 user_api_key_dict: UserAPIKeyAuth | None = None,
472):
473 raise_if_required_body_param_missing(route_type=route_type, data=data)
475 await add_shared_session_to_data(data)
477 raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed())
479 data.pop("enable_tag_filtering", None)
481 team_id: Final = get_team_id_from_data(data)
482 router_model_names: Final = llm_router.model_names if llm_router is not None else []
483 is_proxy_admin_without_team: Final = team_id is None and _is_proxy_admin_request(data)
485 # Preprocess Google GenAI generate content requests
486 if route_type in ["agenerate_content", "agenerate_content_stream"]:
487 # Map generationConfig to config parameter for Google GenAI compatibility
488 if "generationConfig" in data and "config" not in data: 488 ↛ 489line 488 didn't jump to line 489 because the condition on line 488 was never true
489 data["config"] = data.pop("generationConfig")
490 if "api_key" in data or "api_base" in data:
491 if llm_router is not None: 491 ↛ 494line 491 didn't jump to line 494 because the condition on line 491 was always true
492 return getattr(llm_router, f"{route_type}")(**data)
493 else:
494 return getattr(litellm, f"{route_type}")(**data)
496 elif ( 496 ↛ 504line 496 didn't jump to line 504 because the condition on line 496 was never true
497 route_type == "acompletion"
498 and data.get("model", "") is not None
499 and "," in data.get("model", "")
500 and llm_router is not None
501 ):
502 # Handle batch completions with comma-separated models BEFORE user_config check
503 # This ensures batch completion logic is applied even when user_config is set
504 if data.get("fastest_response", False):
505 return llm_router.abatch_completion_fastest_response(**data)
506 else:
507 models: Final = [model.strip() for model in data.pop("model").split(",")]
508 return llm_router.abatch_completion(models=models, **data)
510 elif "user_config" in data: 510 ↛ 511line 510 didn't jump to line 511 because the condition on line 510 was never true
511 return _route_user_config_request(data, route_type)
513 elif "router_settings_override" in data: 513 ↛ 517line 513 didn't jump to line 517 because the condition on line 513 was never true
514 # Apply per-request router settings overrides from key/team config
515 # Instead of creating a new Router (expensive), merge settings into kwargs
516 # The Router already supports per-request overrides for these settings
517 override_settings: Final = data.pop("router_settings_override")
519 # Settings that the Router accepts as per-request kwargs
520 # These override the global router settings for this specific request
521 per_request_settings: Final = [
522 "fallbacks",
523 "context_window_fallbacks",
524 "content_policy_fallbacks",
525 "num_retries",
526 "timeout",
527 "model_group_retry_policy",
528 "routing_strategy",
529 "enable_tag_filtering",
530 ]
532 # Merge override settings into data (only if not already set in request)
533 for key in per_request_settings:
534 if key in override_settings and key not in data:
535 data[key] = override_settings[key]
537 # Use main router with overridden kwargs
538 if llm_router is not None:
539 return getattr(llm_router, f"{route_type}")(**data)
540 else:
541 return getattr(litellm, f"{route_type}")(**data)
542 elif llm_router is not None: 542 ↛ 714line 542 didn't jump to line 714 because the condition on line 542 was always true
543 _raise_if_model_fully_blocked(llm_router=llm_router, model_name=data.get("model"), team_id=team_id)
544 # Evals API: always route to litellm directly (not through router)
545 # But extract model credentials if a model is provided
546 if route_type in [
547 "acreate_eval",
548 "alist_evals",
549 "aget_eval",
550 "aupdate_eval",
551 "adelete_eval",
552 "acancel_eval",
553 "acreate_run",
554 "alist_runs",
555 "aget_run",
556 "acancel_run",
557 "adelete_run",
558 ]:
559 # If a model is provided, get its credentials from the router
560 model: Final = data.get("model")
561 if model and llm_router: 561 ↛ 562line 561 didn't jump to line 562 because the condition on line 561 was never true
562 try:
563 # Try to get deployment credentials for this model
564 deployment_creds = llm_router.get_deployment_credentials(model_id=model)
565 if not deployment_creds:
566 # Try by model group name
567 deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=model)
568 if (
569 deployment
570 and deployment.litellm_params
571 and not llm_router._is_deployment_blocked(deployment)
572 ):
573 deployment_creds = deployment.litellm_params.model_dump(exclude_none=True)
575 # If we found credentials, merge them into data (but don't override user-provided values)
576 if deployment_creds:
577 data.update(deployment_creds)
578 except Exception:
579 # If we can't get deployment creds, continue without them
580 pass
582 return getattr(litellm, f"{route_type}")(**data)
583 # Skip model-based routing for container operations
584 if route_type in [
585 "acreate_container",
586 "alist_containers",
587 "aretrieve_container",
588 "adelete_container",
589 "aupload_container_file",
590 "alist_container_files",
591 "aretrieve_container_file",
592 "adelete_container_file",
593 "aretrieve_container_file_content",
594 ]:
595 return getattr(llm_router, f"{route_type}")(**data)
596 # Interactions API: create with agent, get/delete/cancel don't need model routing
597 if route_type in [
598 "acreate_interaction",
599 "aget_interaction",
600 "adelete_interaction",
601 "acancel_interaction",
602 ]:
603 return getattr(llm_router, f"{route_type}")(**data)
604 # Managed Agents API: these don't need model routing
605 if route_type in [
606 "acreate_agent",
607 "alist_agents",
608 "aget_agent",
609 "adelete_agent",
610 "alist_agent_versions",
611 ]:
612 return getattr(llm_router, f"{route_type}")(**data)
613 if route_type in [
614 "avideo_list",
615 "avideo_status",
616 "avideo_content",
617 "avideo_remix",
618 "avideo_create_character",
619 "avideo_get_character",
620 "avideo_edit",
621 "avideo_extension",
622 "avector_store_file_list",
623 "avector_store_file_retrieve",
624 "avector_store_file_content",
625 "avector_store_file_delete",
626 "acreate_skill",
627 "alist_skills",
628 "aget_skill",
629 "adelete_skill",
630 "aingest",
631 ] and (data.get("model") is None or data.get("model") == ""):
632 # These endpoints don't need a model, use custom_llm_provider directly
633 return getattr(litellm, f"{route_type}")(**data)
635 team_model_name: Final = llm_router.map_team_model(data["model"], team_id) if team_id is not None else None
636 if team_model_name is not None: 636 ↛ 637line 636 didn't jump to line 637 because the condition on line 636 was never true
637 data["model"] = team_model_name
638 return getattr(llm_router, f"{route_type}")(**data)
640 elif ( 640 ↛ 645line 640 didn't jump to line 645 because the condition on line 640 was never true
641 is_proxy_admin_without_team
642 and data["model"] not in router_model_names
643 and data["model"] in llm_router.team_public_model_names
644 ) or llm_router.is_recognized_model(data["model"]):
645 return getattr(llm_router, f"{route_type}")(**data)
647 elif data["model"] not in router_model_names: 647 ↛ 718line 647 didn't jump to line 718 because the condition on line 647 was always true
648 # Check wildcards before checking deployment_names
649 # Priority: 1. Exact model_name match, 2. Wildcard match, 3. deployment_names match
650 if llm_router.router_general_settings.pass_through_all_models: 650 ↛ 651line 650 didn't jump to line 651 because the condition on line 650 was never true
651 return getattr(litellm, f"{route_type}")(**data)
652 elif llm_router.default_deployment is not None or len(llm_router.pattern_router.patterns) > 0: 652 ↛ 653line 652 didn't jump to line 653 because the condition on line 652 was never true
653 return getattr(llm_router, f"{route_type}")(**data)
654 elif data["model"] in llm_router.deployment_names: 654 ↛ 656line 654 didn't jump to line 656 because the condition on line 654 was never true
655 # Only match deployment_names if no wildcard matched
656 return getattr(llm_router, f"{route_type}")(**data, specific_deployment=True)
657 elif route_type in [
658 "amoderation",
659 "aget_responses",
660 "adelete_responses",
661 "acancel_responses",
662 "alist_input_items",
663 "avector_store_create",
664 "avector_store_search",
665 "avector_store_retrieve",
666 "avector_store_list",
667 "avector_store_update",
668 "avector_store_delete",
669 "avector_store_file_create",
670 "avector_store_file_list",
671 "avector_store_file_retrieve",
672 "avector_store_file_content",
673 "avector_store_file_update",
674 "avector_store_file_delete",
675 "asearch",
676 "acreate_container",
677 "alist_containers",
678 "aretrieve_container",
679 "adelete_container",
680 "aupload_container_file",
681 "alist_container_files",
682 "aretrieve_container_file",
683 "adelete_container_file",
684 "aretrieve_container_file_content",
685 ]:
686 # These endpoints can work with or without model parameter
687 return getattr(llm_router, f"{route_type}")(**data)
688 elif route_type in [ 688 ↛ 699line 688 didn't jump to line 699 because the condition on line 688 was never true
689 "avideo_status",
690 "avideo_content",
691 "avideo_remix",
692 "avideo_create_character",
693 "avideo_get_character",
694 "avideo_edit",
695 "avideo_extension",
696 ]:
697 # Video endpoints: If model is provided (e.g., from decoded video_id or target_model_names),
698 # try router first to allow for multi-deployment load balancing
699 try:
700 return getattr(llm_router, f"{route_type}")(**data)
701 except Exception:
702 # If router fails (e.g., model not found in router), fall back to direct call
703 return getattr(litellm, f"{route_type}")(**data)
704 elif _is_a2a_agent_model(data.get("model", "")): 704 ↛ 705line 704 didn't jump to line 705 because the condition on line 704 was never true
705 from litellm.proxy.agent_endpoints.a2a_routing import (
706 route_a2a_agent_request,
707 )
709 result: Final = await route_a2a_agent_request(data, route_type, user_api_key_dict=user_api_key_dict)
710 if result is not None:
711 return result
712 # Fall through to raise exception below if result is None
714 elif user_model is not None or route_type == "allm_passthrough_route":
715 return getattr(litellm, f"{route_type}")(**data)
717 # if no route found then it's a bad request
718 route_name: Final = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type)
719 raise ProxyModelNotFoundError(
720 route=route_name,
721 model_name=data.get("model", ""),
722 )