Coverage for .venv/lib/python3.13/site-packages/litellm/proxy/public_endpoints/public

1import asyncio

2import json

3import os

4import re

5from collections.abc import Awaitable, Callable, Mapping, Sequence

6from importlib.resources import files

7from typing import TYPE_CHECKING, Final, Protocol

9from fastapi import APIRouter, HTTPException, Request

10from pydantic import TypeAdapter

11from typing_extensions import ReadOnly, TypedDict

13import litellm

14from litellm._logging import verbose_logger

15from litellm.litellm_core_utils.get_blog_posts import (

16 BlogPost,

17 BlogPostsResponse,

18 GetBlogPosts,

19 get_blog_posts,

20)

21from litellm.proxy._types import (

22 CommonProxyErrors,

23)

24from litellm.proxy.utils import get_custom_url

25from litellm.repositories.table_repositories import ClaudeCodePluginRepository

26from litellm.router_strategy.complexity_router.fuse_presets import FusePresetCatalog, get_fuse_presets

27from litellm.types.agents import AgentCard

28from litellm.types.mcp import MCPPublicServer

29from litellm.types.proxy.management_endpoints.model_management_endpoints import (

30 ModelGroupInfoProxy,

31)

32from litellm.types.proxy.public_endpoints.public_endpoints import (

33 AgentCreateInfo,

34 AutoRouterPresetRecord,

35 ComplexityScorerDefaults,

36 ProviderCreateInfo,

37 PublicModelHubInfo,

38 SupportedEndpointsResponse,

39)

40from litellm.types.utils import LlmProviders

42if TYPE_CHECKING: 42 ↛ 43line 42 didn't jump to line 43 because the condition on line 42 was never true

43 from datetime import datetime

45router: Final = APIRouter()

48class _ProviderSupportEntry(TypedDict, total=False):

49 display_name: ReadOnly[str]

50 endpoints: ReadOnly[Mapping[str, bool]]

53class _ProvidersFile(TypedDict, total=False):

54 providers: ReadOnly[Mapping[str, _ProviderSupportEntry]]

57class _EndpointProviderEntry(TypedDict):

58 slug: ReadOnly[str]

59 display_name: ReadOnly[str]

62class _EndpointEntry(TypedDict):

63 key: ReadOnly[str]

64 label: ReadOnly[str]

65 endpoint: ReadOnly[str]

66 providers: ReadOnly[Sequence[_EndpointProviderEntry]]

69class _PluginRow(Protocol):

70 @property

71 def id(self) -> str: ... 71 ↛ exitline 71 didn't return from function 'id' because

73 @property

74 def name(self) -> str: ... 74 ↛ exitline 74 didn't return from function 'name' because

76 @property

77 def enabled(self) -> bool: ... 77 ↛ exitline 77 didn't return from function 'enabled' because

79 @property

80 def created_at(self) -> "datetime | None": ... 80 ↛ exitline 80 didn't return from function 'created_at' because

82 @property

83 def updated_at(self) -> "datetime | None": ... 83 ↛ exitline 83 didn't return from function 'updated_at' because

85 @property

86 def manifest_json(self) -> str | None: ... 86 ↛ exitline 86 didn't return from function 'manifest_json' because

89class _PluginTableActions(Protocol):

90 def find_many(self, *, where: Mapping[str, bool]) -> Awaitable[Sequence[_PluginRow]]: ... 90 ↛ exitline 90 didn't return from function 'find_many' because

93def _plugin_table(prisma_client: object) -> _PluginTableActions:

94 return ClaudeCodePluginRepository(prisma_client).table

97# ---------------------------------------------------------------------------

98# /public/endpoints — helpers

99# ---------------------------------------------------------------------------

100

101_ENDPOINT_METADATA: Final[Mapping[str, Mapping[str, str]]] = {

102 "chat_completions": {"label": "Chat Completions", "endpoint": "/chat/completions"},

103 "messages": {"label": "Messages", "endpoint": "/messages"},

104 "responses": {"label": "Responses", "endpoint": "/responses"},

105 "embeddings": {"label": "Embeddings", "endpoint": "/embeddings"},

106 "image_generations": {

107 "label": "Image Generations",

108 "endpoint": "/images/generations",

109 },

110 "audio_transcriptions": {

111 "label": "Audio Transcriptions",

112 "endpoint": "/audio/transcriptions",

113 },

114 "audio_speech": {"label": "Audio Speech", "endpoint": "/audio/speech"},

115 "moderations": {"label": "Moderations", "endpoint": "/moderations"},

116 "batches": {"label": "Batches", "endpoint": "/batches"},

117 "rerank": {"label": "Rerank", "endpoint": "/rerank"},

118 "ocr": {"label": "OCR", "endpoint": "/ocr"},

119 "search": {"label": "Search", "endpoint": "/search"},

120 "skills": {"label": "Skills", "endpoint": "/skills"},

121 "interactions": {"label": "Interactions", "endpoint": "/interactions"},

122 "a2a": {"label": "A2A (Agent Gateway)", "endpoint": "/a2a/{agent}/message/send"},

123 "container": {"label": "Containers", "endpoint": "/containers"},

124 "container_files": {

125 "label": "Container Files",

126 "endpoint": "/containers/{id}/files",

127 },

128 "compact": {"label": "Compact", "endpoint": "/responses/compact"},

129 "files": {"label": "Files", "endpoint": "/files"},

130 "image_edits": {"label": "Image Edits", "endpoint": "/images/edits"},

131 "vector_stores_create": {

132 "label": "Vector Stores (Create)",

133 "endpoint": "/vector_stores",

134 },

135 "vector_stores_search": {

136 "label": "Vector Stores (Search)",

137 "endpoint": "/vector_stores/{id}/search",

138 },

139 "vector_store_files": {

140 "label": "Vector Store Files",

141 "endpoint": "/vector_stores/{id}/files",

142 },

143 "video_generations": {

144 "label": "Video Generations",

145 "endpoint": "/videos/generations",

146 },

147 "assistants": {"label": "Assistants", "endpoint": "/assistants"},

148 "fine_tuning": {"label": "Fine Tuning", "endpoint": "/fine_tuning/jobs"},

149 "text_completion": {"label": "Text Completion", "endpoint": "/completions"},

150 "realtime": {"label": "Realtime", "endpoint": "/realtime"},

151 "count_tokens": {"label": "Count Tokens", "endpoint": "/utils/token_counter"},

152 "image_variations": {"label": "Image Variations", "endpoint": "/images/variations"},

153 "generateContent": {"label": "Generate Content", "endpoint": "/generateContent"},

154 "bedrock_invoke": {"label": "Bedrock Invoke", "endpoint": "/bedrock/invoke"},

155 "bedrock_converse": {"label": "Bedrock Converse", "endpoint": "/bedrock/converse"},

156 "rag_ingest": {"label": "RAG Ingest", "endpoint": "/rag/ingest"},

157 "rag_query": {"label": "RAG Query", "endpoint": "/rag/query"},

158}

159

160_SLUG_SUFFIX_RE: Final = re.compile(r"\s*$`[^`]+`$\s*$")

161

162# Loaded once on first request; never invalidated (local file, no TTL needed).

163_cached_endpoints: SupportedEndpointsResponse | None = None

164

165

166def _clean_display_name(raw: str) -> str:

167 return _SLUG_SUFFIX_RE.sub("", raw).strip()

168

169

170def _build_endpoints(raw: _ProvidersFile) -> list[_EndpointEntry]:

171 """Transform raw provider_endpoints_support_backup.json into the response shape."""

172 providers: Final = raw.get("providers", {})

173

174 # Collect endpoint keys in insertion order (union across all providers).

175 seen: Final[set[str]] = set()

176 all_keys: Final[list[str]] = []

177 for provider_data in providers.values():

178 for key in provider_data.get("endpoints", {}):

179 if key not in seen:

180 seen.add(key)

181 all_keys.append(key)

182

183 result: Final[list[_EndpointEntry]] = []

184 for key in all_keys:

185 meta = _ENDPOINT_METADATA.get(key)

186 label = meta["label"] if meta else key.replace("_", " ").title()

187 path = meta["endpoint"] if meta else "/" + key.replace("_", "/")

188

189 supporting: list[_EndpointProviderEntry] = [

190 {

191 "slug": slug,

192 "display_name": _clean_display_name(pd.get("display_name", slug)),

193 }

194 for slug, pd in providers.items()

195 if pd.get("endpoints", {}).get(key)

196 ]

197 result.append({"key": key, "label": label, "endpoint": path, "providers": supporting})

198

199 return result

200

201

202_PROVIDERS_FILE_ADAPTER: Final = TypeAdapter(_ProvidersFile)

203_PROVIDER_CREATE_FIELDS_ADAPTER: Final = TypeAdapter(list[ProviderCreateInfo])

204

205

206def _load_endpoints() -> list[_EndpointEntry]:

207 raw: Final = _PROVIDERS_FILE_ADAPTER.validate_python(

208 json.loads(files("litellm").joinpath("provider_endpoints_support_backup.json").read_text(encoding="utf-8"))

209 )

210 return _build_endpoints(raw)

211

212

213# ---------------------------------------------------------------------------

214

215

216@router.get(

217 "/public/model_hub",

218 tags=["public", "model management"],

219 response_model=list[ModelGroupInfoProxy],

220)

221async def public_model_hub():

222 import litellm

223 from litellm.proxy.health_endpoints._health_endpoints import (

224 _convert_health_check_to_dict,

225 )

226 from litellm.proxy.proxy_server import (

227 _get_model_group_info,

228 llm_router,

229 prisma_client,

230 )

231

232 if llm_router is None: 232 ↛ 233line 232 didn't jump to line 233 because the condition on line 232 was never true

233 raise HTTPException(status_code=400, detail=CommonProxyErrors.no_llm_router.value)

234

235 model_groups: list[ModelGroupInfoProxy] = []

236 if litellm.public_model_groups is not None: 236 ↛ 244line 236 didn't jump to line 244 because the condition on line 236 was always true

237 model_groups = _get_model_group_info(

238 llm_router=llm_router,

239 all_models_str=litellm.public_model_groups,

240 model_group=None,

241 )

242

243 # Fetch health check information if available

244 health_checks_map: Final = {}

245 if prisma_client is not None: 245 ↛ 258line 245 didn't jump to line 258 because the condition on line 245 was always true

246 try:

247 latest_checks: Final = await prisma_client.get_all_latest_health_checks()

248 for check in latest_checks:

249 key = check.model_id if check.model_id else check.model_name

250 if key: 250 ↛ 251line 250 didn't jump to line 251 because the condition on line 250 was never true

251 health_check_dict = _convert_health_check_to_dict(check)

252 health_checks_map[key] = health_check_dict

253 if check.model_name:

254 health_checks_map[check.model_name] = health_check_dict

255 except Exception:

256 pass

257

258 for model_group in model_groups:

259 health_info = health_checks_map.get(model_group.model_group)

260 if health_info: 260 ↛ 261line 260 didn't jump to line 261 because the condition on line 260 was never true

261 model_group.health_status = health_info.get("status")

262 model_group.health_response_time = health_info.get("response_time_ms")

263 model_group.health_checked_at = health_info.get("checked_at")

264

265 return model_groups

266

267

268@router.get(

269 "/public/agent_hub",

270 tags=["[beta] Agents", "public"],

271 response_model=list[AgentCard],

272)

273async def get_agents(request: Request):

274 import litellm

275 from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry

276

277 agents: Final = global_agent_registry.get_public_agent_list()

278

279 if litellm.public_agent_groups is None:

280 return []

281

282 return [

283 {

284 **(agent.agent_card_params or {}),

285 "url": get_custom_url(str(request.base_url), route=f"a2a/{agent.agent_id}"),

286 }

287 for agent in agents

288 if not global_agent_registry.ids_for_agent(agent.agent_id).isdisjoint(litellm.public_agent_groups)

289 ]

290

291

292@router.get(

293 "/public/mcp_hub",

294 tags=["[beta] MCP", "public"],

295 response_model=list[MCPPublicServer],

296)

297async def get_mcp_servers():

298 from litellm.proxy._experimental.mcp_server.mcp_server_manager import (

299 global_mcp_server_manager,

300 )

301

302 public_mcp_servers: Final = global_mcp_server_manager.get_public_mcp_servers()

303 return [MCPPublicServer.model_validate(server.model_dump()) for server in public_mcp_servers]

304

305

306@router.get(

307 "/public/skill_hub",

308 tags=["public", "Claude Code Marketplace"],

309)

310async def public_skill_hub():

311 """Return enabled (public) Claude Code skills — no auth required."""

312 from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import (

313 _get_prisma_client,

314 )

315 from litellm.types.proxy.claude_code_endpoints import (

316 ListPluginsResponse,

317 PluginListItem,

318 )

319

320 try:

321 prisma_client: Final = await _get_prisma_client()

322 plugins: Final = await _plugin_table(prisma_client).find_many(where={"enabled": True})

323 items: Final = []

324 for plugin in plugins: 324 ↛ 325line 324 didn't jump to line 325 because the loop on line 324 never started

325 raw = plugin.manifest_json or {}

326 manifest = json.loads(raw) if isinstance(raw, str) else raw

327 items.append(

328 PluginListItem(

329 id=plugin.id,

330 name=plugin.name,

331 enabled=plugin.enabled,

332 created_at=str(plugin.created_at) if plugin.created_at else None,

333 updated_at=str(plugin.updated_at) if plugin.updated_at else None,

334 source=manifest.get("source", {}),

335 description=manifest.get("description"),

336 version=manifest.get("version"),

337 category=manifest.get("category"),

338 keywords=manifest.get("keywords"),

339 author=manifest.get("author"),

340 homepage=manifest.get("homepage"),

341 domain=manifest.get("domain"),

342 namespace=manifest.get("namespace"),

343 )

344 )

345 return ListPluginsResponse(plugins=items, count=len(items))

346 except Exception as e:

347 raise HTTPException(status_code=500, detail=str(e))

348

349

350@router.get(

351 "/public/model_hub/info",

352 tags=["public", "model management"],

353 response_model=PublicModelHubInfo,

354)

355async def public_model_hub_info():

356 import litellm

357 from litellm.proxy.proxy_server import _title, version

358

359 try:

360 from litellm_enterprise.proxy.proxy_server import EnterpriseProxyConfig

361

362 custom_docs_description = EnterpriseProxyConfig.get_custom_docs_description()

363 except Exception:

364 custom_docs_description = None

365

366 return PublicModelHubInfo(

367 docs_title=_title,

368 custom_docs_description=custom_docs_description,

369 litellm_version=version,

370 useful_links=litellm.public_model_groups_links,

371 )

372

373

374@router.get(

375 "/public/providers",

376 tags=["public", "providers"],

377 response_model=list[str],

378)

379async def get_supported_providers() -> list[str]:

380 """

381 Return a sorted list of all providers supported by LiteLLM.

382 """

383

384 return sorted(provider.value for provider in LlmProviders)

385

386

387@router.get(

388 "/public/providers/fields",

389 tags=["public", "providers"],

390 response_model=list[ProviderCreateInfo],

391)

392async def get_provider_fields() -> list[ProviderCreateInfo]:

393 """

394 Return provider metadata required by the dashboard create-model flow.

395 """

396

397 provider_create_fields_path: Final = os.path.join(

398 os.path.dirname(os.path.dirname(os.path.dirname(__file__))),

399 "proxy",

400 "public_endpoints",

401 "provider_create_fields.json",

402 )

403

404 with open(provider_create_fields_path, "r") as f:

405 provider_create_fields: Final = _PROVIDER_CREATE_FIELDS_ADAPTER.validate_python(json.load(f))

406

407 return provider_create_fields

408

409

410@router.get(

411 "/public/complexity_router/scorer_defaults",

412 tags=["public", "auto router"],

413 response_model=ComplexityScorerDefaults,

414)

415async def get_complexity_scorer_defaults() -> ComplexityScorerDefaults:

416 """

417 Return the complexity router's shipped heuristic scorer defaults, for the dashboard to prefill with.

418 """

419 from litellm.router_strategy.complexity_router.config import (

420 DEFAULT_DIMENSION_WEIGHTS,

421 DEFAULT_TIER_BOUNDARIES,

422 DEFAULT_TOKEN_THRESHOLDS,

423 )

424

425 return ComplexityScorerDefaults(

426 tier_boundaries=DEFAULT_TIER_BOUNDARIES,

427 token_thresholds=DEFAULT_TOKEN_THRESHOLDS,

428 dimension_weights=DEFAULT_DIMENSION_WEIGHTS,

429 )

430

431

432@router.get(

433 "/public/complexity_router/fuse_presets",

434 response_model=FusePresetCatalog,

435)

436async def get_public_fuse_presets() -> FusePresetCatalog:

437 return get_fuse_presets()

438

439

440@router.get(

441 "/public/litellm_model_cost_map",

442 tags=["public", "model management"],

443)

444async def get_litellm_model_cost_map():

445 """

446 Public endpoint to get the LiteLLM model cost map.

447 Returns pricing information for all supported models.

448 """

449 import litellm

450

451 try:

452 _model_cost_map: Final = litellm.model_cost

453 return _model_cost_map

454 except Exception as e:

455 raise HTTPException(

456 status_code=500,

457 detail=f"Internal Server Error ({e})",

458 )

459

460

461@router.get(

462 "/public/litellm_blog_posts",

463 tags=["public"],

464 response_model=BlogPostsResponse,

465)

466async def get_litellm_blog_posts():

467 """

468 Public endpoint to get the latest LiteLLM blog posts.

469

470 Fetches from GitHub with a 1-hour in-process cache.

471 Falls back to the bundled local backup on any failure.

472 """

473 try:

474 posts_data = get_blog_posts(url=litellm.blog_posts_url)

475 except Exception as e:

476 verbose_logger.warning("LiteLLM: get_litellm_blog_posts endpoint fallback triggered: %s", str(e))

477 posts_data = GetBlogPosts.load_local_blog_posts()

478

479 posts: Final = [BlogPost(**p) for p in posts_data[:5]]

480 return BlogPostsResponse(posts=posts)

481

482

483_AUTOROUTER_PRESETS_ADAPTER: Final = TypeAdapter(dict[str, AutoRouterPresetRecord])

484

485

486def _load_bundled_autorouter_presets() -> Mapping[str, AutoRouterPresetRecord]:

487 raw: Final = json.loads(

488 files("litellm.proxy.public_endpoints").joinpath("autorouter_presets.json").read_text(encoding="utf-8")

489 )

490 return _AUTOROUTER_PRESETS_ADAPTER.validate_python(raw)

491

492

493async def _fetch_remote_autorouter_presets(url: str) -> Mapping[str, AutoRouterPresetRecord]:

494 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client

495 from litellm.types.llms.custom_http import httpxSpecialProvider

496

497 client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.UI)

498 response: Final = await client.get(url, timeout=5.0)

499 response.raise_for_status()

500 presets: Final = _AUTOROUTER_PRESETS_ADAPTER.validate_python(response.json())

501 if not presets: 501 ↛ 502line 501 didn't jump to line 502 because the condition on line 501 was never true

502 raise ValueError("remote auto-router preset catalog is empty")

503 return presets

504

505

506async def _resolve_autorouter_presets(

507 url: str,

508 fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]],

509) -> Mapping[str, AutoRouterPresetRecord]:

510 if os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true": 510 ↛ 511line 510 didn't jump to line 511 because the condition on line 510 was never true

511 return _load_bundled_autorouter_presets()

512 try:

513 return await fetch(url)

514 except Exception as e:

515 verbose_logger.warning(

516 "LiteLLM: failed to fetch auto-router presets from %s: %s. Serving the bundled catalog for the life of this process.",

517 url,

518 str(e),

519 )

520 return _load_bundled_autorouter_presets()

521

522

523class _AutoRouterPresetsCache:

524 presets: Mapping[str, AutoRouterPresetRecord] | None = None

525 lock: asyncio.Lock | None = None

526

527

528async def get_autorouter_presets(

529 url: str,

530 fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]] = _fetch_remote_autorouter_presets,

531) -> Mapping[str, AutoRouterPresetRecord]:

532 cached: Final = _AutoRouterPresetsCache.presets

533 if cached is not None:

534 return cached

535 if _AutoRouterPresetsCache.lock is None: 535 ↛ 537line 535 didn't jump to line 537 because the condition on line 535 was always true

536 _AutoRouterPresetsCache.lock = asyncio.Lock()

537 async with _AutoRouterPresetsCache.lock:

538 held: Final = _AutoRouterPresetsCache.presets

539 if held is not None: 539 ↛ 540line 539 didn't jump to line 540 because the condition on line 539 was never true

540 return held

541 resolved: Final = await _resolve_autorouter_presets(url=url, fetch=fetch)

542 _AutoRouterPresetsCache.presets = resolved

543 return resolved

544

545

546@router.get(

547 "/public/autorouter_presets",

548 tags=["public", "auto router"], # mutable-ok: FastAPI route tags take a list

549 response_model=dict[str, AutoRouterPresetRecord],

550)

551async def get_public_autorouter_presets() -> Mapping[str, AutoRouterPresetRecord]:

552 """

553 Return the auto-router preset catalog the dashboard's template picker renders.

554

555 Resolved once per process, like the model cost map: fetched from ``litellm.autorouter_presets_url``

556 (override with ``LITELLM_AUTOROUTER_PRESETS_URL``) on the first request, falling back to the

557 catalog bundled with the package on any failure. Set ``LITELLM_LOCAL_AUTOROUTER_PRESETS=True``

558 to serve the bundled catalog only. A restart picks up a newly published catalog.

559 """

560 return await get_autorouter_presets(url=litellm.autorouter_presets_url)

561

562

563@router.get(

564 "/public/endpoints",

565 tags=["public"],

566 response_model=SupportedEndpointsResponse,

567)

568async def get_supported_endpoints() -> SupportedEndpointsResponse:

569 """

570 Return the list of LiteLLM proxy endpoints and which providers support each one.

571

572 Reads from the bundled local backup file. Result is cached in-process for

573 the lifetime of the server process.

574 """

575 global _cached_endpoints

576 if _cached_endpoints is None:

577 _cached_endpoints = SupportedEndpointsResponse(endpoints=_load_endpoints())

578 return _cached_endpoints

579

580

581@router.get(

582 "/public/agents/fields",

583 tags=["public", "[beta] Agents"],

584 response_model=list[AgentCreateInfo],

585)

586async def get_agent_fields() -> list[AgentCreateInfo]:

587 """

588 Return agent type metadata required by the dashboard create-agent flow.

589

590 If an agent has `inherit_credentials_from_provider`, the provider's credential

591 fields are automatically appended to the agent's credential_fields.

592 """

593 base_path: Final = os.path.join(

594 os.path.dirname(os.path.dirname(os.path.dirname(__file__))),

595 "proxy",

596 "public_endpoints",

597 )

598

599 agent_create_fields_path: Final = os.path.join(base_path, "agent_create_fields.json")

600 provider_create_fields_path: Final = os.path.join(base_path, "provider_create_fields.json")

601

602 with open(agent_create_fields_path, "r") as f:

603 agent_create_fields: Final = json.load(f)

604

605 with open(provider_create_fields_path, "r") as f:

606 provider_create_fields: Final = json.load(f)

607

608 # Build a lookup map for providers by name

609 provider_map: Final = {p["provider"]: p for p in provider_create_fields}

610

611 # Merge inherited credential fields

612 for agent in agent_create_fields:

613 inherit_from = agent.get("inherit_credentials_from_provider")

614 if inherit_from and inherit_from in provider_map:

615 provider = provider_map[inherit_from]

616 # Copy provider fields and mark them for inclusion in litellm_params

617 inherited_fields = []

618 for field in provider.get("credential_fields", []):

619 field_copy = field.copy()

620 field_copy["include_in_litellm_params"] = True

621 inherited_fields.append(field_copy)

622 # Append provider credential fields after agent's own fields

623 agent["credential_fields"] = agent.get("credential_fields", []) + inherited_fields

624 # Remove the inherit field from response (not needed by frontend)

625 agent.pop("inherit_credentials_from_provider", None)

626

627 return agent_create_fields

Coverage for .venv/lib/python3.13/site-packages/litellm/proxy/public_endpoints/public_endpoints.py: 84%

251 statements