Coverage for .venv/lib/python3.13/site-packages/litellm/proxy/management_endpoints/model_management_endpoints.py: 44%
1016 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-10 12:01 +0000
1"""
2Allow proxy admin to add/update/delete models in the db
4Currently most endpoints are in `proxy_server.py`, but those should be moved here over time.
6Endpoints here:
8model/{model_id}/update - PATCH endpoint for model update.
9"""
11#### MODEL MANAGEMENT ####
13import asyncio
14import datetime
15import json
16from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence
17from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress
18from dataclasses import dataclass
19from fnmatch import fnmatchcase
20from json import JSONDecodeError
21from types import MappingProxyType
22from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable
24from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
25from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator
27import litellm
28from litellm._logging import verbose_proxy_logger
29from litellm._uuid import uuid
30from litellm.constants import LITELLM_PROXY_ADMIN_NAME
31from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
32from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
33from litellm.litellm_core_utils.ptu_pricing import (
34 CUSTOM_PRICING_FIELDS,
35 PTU_EMPTIED_PRICING_FIELDS,
36 PTU_MODEL_INFO_FIELDS,
37 PTU_ZEROED_PRICING_FIELDS,
38 PTU_ZEROED_TABLE_FIELDS,
39 SEARCH_CONTEXT_SIZES,
40 ptu_config_error,
41)
42from litellm.proxy._types import (
43 BlockModelRequest,
44 CommonProxyErrors,
45 LiteLLM_ProxyModelTable,
46 LiteLLM_TeamTable,
47 LitellmTableNames,
48 LitellmUserRoles,
49 ModelInfoDelete,
50 PrismaCompatibleUpdateDBModel,
51 ProxyErrorTypes,
52 ProxyException,
53 ReconcileOutcome,
54 TeamModelAddRequest,
55 TeamModelDeleteRequest,
56 UserAPIKeyAuth,
57)
58from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY
59from litellm.proxy.auth.team_grants import team_model_aliases
60from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
61from litellm.proxy.common_utils.config_sync_pubsub import (
62 coordination_redis_cache,
63 publish_config_change,
64)
65from litellm.proxy.common_utils.encrypt_decrypt_utils import (
66 decrypt_value_helper,
67 encrypt_value_helper,
68)
69from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
70from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
71from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin
72from litellm.proxy.management_endpoints.team_endpoints import (
73 _refresh_cached_team,
74 append_team_models,
75 team_model_add,
76 team_model_delete,
77)
78from litellm.proxy.management_endpoints.team_endpoints import (
79 update_team as _legacy_update_team,
80)
81from litellm.proxy.management_helpers.access_group_model_sync import (
82 sync_access_groups_for_deleted_model,
83 sync_access_groups_for_renamed_model,
84)
85from litellm.proxy.management_helpers.audit_logs import create_object_audit_log
86from litellm.proxy.management_helpers.auto_router_permissions import (
87 MemberAutoRouterWrite,
88 StoredAutoRouterIdentity,
89 authorize_member_auto_router_dependencies,
90 authorize_member_auto_router_team,
91 authorize_member_auto_router_write,
92)
93from litellm.proxy.management_helpers.model_allowlist_rename_sync import sync_model_allowlists_for_renamed_model
94from litellm.proxy.spend_tracking.ptu_feature_flag import (
95 PTU_COST_ATTRIBUTION_ENV_VAR,
96 is_ptu_cost_attribution_enabled,
97)
98from litellm.proxy.utils import PrismaClient, ProxyLogging
99from litellm.repositories.credentials_repository import CredentialsRepository
100from litellm.repositories.model_repository import ModelRepository
101from litellm.repositories.prisma_protocols import TableActions
102from litellm.repositories.table_repositories import ModelTableRepository
103from litellm.repositories.team_repository import TeamRepository
104from litellm.router import Router
105from litellm.router_strategy.complexity_router import (
106 DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
107 ClassificationRubric,
108 ComplexityRouterConfig,
109 ComplexityTier,
110 TierDefinition,
111 built_in_tier_classification_prompt,
112 classification_system_prompt,
113 custom_tier_classification_prompt,
114 normalize_classification_examples,
115 normalize_classification_prompt,
116)
117from litellm.router_utils.auto_router_model_naming import (
118 GATED_AUTO_ROUTER_CAPABILITIES,
119 STRATEGY_ROUTER_PARAM_FIELDS,
120 capability_limit_violation,
121 carries_complexity_router_settings,
122 count_capability_routers,
123 gated_capability_of,
124 is_complexity_router_model,
125 validate_complexity_router_config_placement,
126 validate_complexity_router_config_write,
127 validate_strategy_router_model_write,
128)
129from litellm.router_utils.auto_router_tuning_baseline import is_mutable_tuned_candidate, tuning_quota_violation
130from litellm.types.llms.bedrock import AwsSessionTag
131from litellm.types.proxy.management_endpoints.model_management_endpoints import (
132 AutoRouterClassifierDefaultPromptResponse,
133 UpdateUsefulLinksRequest,
134)
135from litellm.types.router import (
136 SPECIAL_MODEL_INFO_PARAMS,
137 Deployment,
138 GenericLiteLLMParams,
139 ModelInfo,
140 updateDeployment,
141 updateLiteLLMParams,
142)
143from litellm.types.utils import (
144 COST_MAP_LOOKUP_KEY,
145 echoed_cost_map_fields,
146 echoed_cost_map_pricing_fields,
147 without_server_derived_pricing,
148)
149from litellm.utils import get_utc_datetime
151if TYPE_CHECKING: 151 ↛ 152line 151 didn't jump to line 152 because the condition on line 151 was never true
152 from prisma import models as prisma_models
153 from prisma import types as prisma_types
155router: Final = APIRouter()
156CLEARABLE_LITELLM_PARAMS: Final = frozenset({"cache_control_injection_points", "litellm_credential_name"})
157NULL_CLEARABLE_LITELLM_PARAMS: Final = frozenset((*SPECIAL_MODEL_INFO_PARAMS, *CLEARABLE_LITELLM_PARAMS))
160async def update_team(*args, **kwargs):
161 """
162 Backward-compatible shim for tests/legacy call sites that patch this symbol.
163 Team model management now uses team_model_add/team_model_delete directly.
164 """
165 return await _legacy_update_team(*args, **kwargs)
168class UpdatePublicModelGroupsRequest(BaseModel):
169 """Request model for updating public model groups"""
171 model_groups: list[str] = Field(description="List of model group names to make public")
173 model_config = ConfigDict(extra="forbid")
176class _ProxyModelRow(Protocol):
177 @property
178 def model_id(self) -> str: ... 178 ↛ exitline 178 didn't return from function 'model_id' because
180 @property
181 def model_name(self) -> str: ... 181 ↛ exitline 181 didn't return from function 'model_name' because
183 @property
184 def model_info(self) -> object: ... 184 ↛ exitline 184 didn't return from function 'model_info' because
186 def model_dump_json(self, *, exclude_none: bool = False) -> str: ... 186 ↛ exitline 186 didn't return from function 'model_dump_json' because
189class _ProxyModelTable(Protocol):
190 def find_unique(self, *, where: Mapping[str, object]) -> Awaitable[BaseModel | None]: ... 190 ↛ exitline 190 didn't return from function 'find_unique' because
192 def find_many(self, *, where: Mapping[str, object]) -> Awaitable[Sequence[_ProxyModelRow]]: ... 192 ↛ exitline 192 didn't return from function 'find_many' because
194 def create(self, *, data: Mapping[str, object]) -> Awaitable[_ProxyModelRow]: ... 194 ↛ exitline 194 didn't return from function 'create' because
196 def update( 196 ↛ exitline 196 didn't return from function 'update' because
197 self, *, where: Mapping[str, object], data: Mapping[str, object]
198 ) -> Awaitable[_ProxyModelRow | None]: ...
200 def delete(self, *, where: Mapping[str, object]) -> Awaitable[_ProxyModelRow | None]: ... 200 ↛ exitline 200 didn't return from function 'delete' because
202 def delete_many(self, *, where: Mapping[str, object]) -> Awaitable[int]: ... 202 ↛ exitline 202 didn't return from function 'delete_many' because
205class _TxModelTables(Protocol):
206 litellm_proxymodeltable: _ProxyModelTable
208 async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... 208 ↛ exitline 208 didn't return from function 'query_raw' because
211@runtime_checkable
212class _TransactionFactory(Protocol):
213 def __call__(self, *, timeout: datetime.timedelta = ...) -> AbstractAsyncContextManager[_TxModelTables]: ... 213 ↛ exitline 213 didn't return from function '__call__' because
216class _ModelTransactionClient(BaseModel):
217 model_config = ConfigDict(arbitrary_types_allowed=True, from_attributes=True)
219 tx: _TransactionFactory
222@dataclass(frozen=True, slots=True)
223class _TransactionClient:
224 db: _TxModelTables
227_RowT = TypeVar("_RowT")
230class _ExistingModelRow(Protocol):
231 @property
232 def litellm_params(self) -> Mapping[str, object]: ... 232 ↛ exitline 232 didn't return from function 'litellm_params' because
234 def model_dump_json(self, *, exclude_none: bool = False) -> str: ... 234 ↛ exitline 234 didn't return from function 'model_dump_json' because
237class _TeamRow(Protocol):
238 @property
239 def models(self) -> Sequence[str]: ... 239 ↛ exitline 239 didn't return from function 'models' because
241 def model_dump(self) -> Mapping[str, object]: ... 241 ↛ exitline 241 didn't return from function 'model_dump' because
244class _TeamLookupTable(Protocol):
245 def find_unique(self, *, where: Mapping[str, object]) -> Awaitable[_TeamRow | None]: ... 245 ↛ exitline 245 didn't return from function 'find_unique' because
248class _TeamTable(_TeamLookupTable, Protocol):
249 def update( 249 ↛ exitline 249 didn't return from function 'update' because
250 self, *, where: Mapping[str, object], data: Mapping[str, object], include: Mapping[str, bool]
251 ) -> Awaitable[LiteLLM_TeamTable]: ...
254def _proxy_model_table(prisma_client: PrismaClient) -> _ProxyModelTable:
255 return ModelRepository(prisma_client).table
258def _repo_team_table(prisma_client: PrismaClient) -> _TeamLookupTable:
259 return TeamRepository(WriterPinnedClient(prisma_client.db)).table
262def _db_team_table(prisma_client: PrismaClient) -> _TeamTable:
263 return prisma_client.db.litellm_teamtable
266def _model_alias_table(prisma_client: PrismaClient) -> "TableActions[prisma_models.LiteLLM_ModelTable]":
267 return ModelTableRepository(prisma_client).table
270async def get_db_model(model_id: str, prisma_client: PrismaClient) -> Deployment | None:
271 db_model: Final = await _proxy_model_table(prisma_client).find_unique(where={"model_id": model_id})
273 if not db_model: 273 ↛ 276line 273 didn't jump to line 276 because the condition on line 273 was always true
274 return None
276 deployment_pydantic_obj: Final = Deployment(**db_model.model_dump(exclude_none=True))
277 return deployment_pydantic_obj
280def _strategy_router_write_violation(
281 incoming_params: GenericLiteLLMParams | None,
282 existing_params: GenericLiteLLMParams | None,
283) -> str | None:
284 """Reject writes that would corrupt a strategy router's pseudo-model.
286 An auto-router deployment's ``litellm_params.model`` (``auto_router/...``) is
287 the discriminator the router loads it by; a write that mangles it makes the
288 router drop the deployment silently under ``ignore_invalid_deployments``.
289 A patch adding auto-router settings is judged against the effective model,
290 decrypting the stored model when the patch omits it, so a regular deployment
291 cannot claim a strategy-router configuration. Unrelated partial patches and
292 restores that do not touch strategy-router settings stay legal. A config is
293 judged only when the write carries one, for the same reason: a rename must
294 not be held hostage by a stored config it does not touch. Returns the
295 violation, or None.
296 """
297 if incoming_params is None: 297 ↛ 298line 297 didn't jump to line 298 because the condition on line 297 was never true
298 return None
299 config_violation: Final = validate_complexity_router_config_write(
300 complexity_router_config=(
301 _effective_complexity_router_config(incoming_params, existing_params)
302 if incoming_params.complexity_router_config is not None
303 else None
304 )
305 )
306 if config_violation is not None: 306 ↛ 307line 306 didn't jump to line 307 because the condition on line 306 was never true
307 return config_violation
308 present_fields: Final = frozenset(
309 field
310 for field in STRATEGY_ROUTER_PARAM_FIELDS
311 for source in (incoming_params, existing_params)
312 if source is not None and getattr(source, field, None) is not None
313 )
314 effective_params: Final = _effective_complexity_router_params(incoming_params, existing_params)
315 effective_model: Final = effective_params.get("model")
316 if carries_complexity_router_settings(
317 effective_model if isinstance(effective_model, str) else None, present_fields
318 ):
319 placement_violation: Final = validate_complexity_router_config_placement(incoming_params.model_extra)
320 if placement_violation is not None: 320 ↛ 321line 320 didn't jump to line 321 because the condition on line 320 was never true
321 return placement_violation
322 return validate_strategy_router_model_write(
323 model=effective_model if isinstance(effective_model, str) else "",
324 present_fields=present_fields,
325 )
328def _raise_on_strategy_router_write_violation(
329 incoming_params: GenericLiteLLMParams | None,
330 existing_params: GenericLiteLLMParams | None,
331) -> None:
332 violation = _strategy_router_write_violation(incoming_params=incoming_params, existing_params=existing_params)
333 if violation is None:
334 return
335 raise ProxyException(
336 message=violation,
337 type=ProxyErrorTypes.validation_error.value,
338 code=status.HTTP_400_BAD_REQUEST,
339 param="litellm_params.model",
340 )
343def _stored_credential_name(existing_litellm_params: GenericLiteLLMParams | None) -> str | None:
344 if existing_litellm_params is None or existing_litellm_params.litellm_credential_name is None:
345 return None
346 return decrypt_value_helper(
347 value=existing_litellm_params.litellm_credential_name,
348 key="litellm_credential_name",
349 exception_type="debug",
350 return_original_value=True,
351 )
354async def _raise_on_invalid_credential_name(
355 litellm_params: updateLiteLLMParams | None,
356 existing_litellm_params: GenericLiteLLMParams | None,
357 prisma_client: PrismaClient,
358) -> None:
359 if litellm_params is None or "litellm_credential_name" not in litellm_params.model_fields_set:
360 return
361 credential_name: Final = litellm_params.litellm_credential_name
362 if credential_name is None:
363 return
364 if credential_name == "":
365 raise ProxyException(
366 message="litellm_credential_name cannot be an empty string. Send null to detach the stored credential or omit the field to leave it unchanged.",
367 type=ProxyErrorTypes.validation_error.value,
368 code=status.HTTP_400_BAD_REQUEST,
369 param="litellm_credential_name",
370 )
371 if credential_name == _stored_credential_name(existing_litellm_params):
372 return
373 if CredentialAccessor.find_credential(credential_name) is not None:
374 return
375 stored_credential: Final = await CredentialsRepository(WriterPinnedClient(prisma_client.db)).find_by_name(
376 credential_name
377 )
378 if stored_credential is not None:
379 return
380 raise ProxyException(
381 message=f"Credential '{credential_name}' not found. Create it via /credentials before attaching it to a model.",
382 type=ProxyErrorTypes.validation_error.value,
383 code=status.HTTP_400_BAD_REQUEST,
384 param="litellm_credential_name",
385 )
388AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY: Final = 5_872_301
389_CAPABILITY_LOCK_SQL: Final = "SELECT 1 AS locked FROM pg_advisory_xact_lock($1)"
390_STORED_LITELLM_PARAMS_SQL: Final = (
391 "(CASE jsonb_typeof(litellm_params) WHEN 'string' THEN (litellm_params #>> '{}')::jsonb ELSE litellm_params END)"
392)
393_STORED_COMPLEXITY_CONFIG_SQL: Final = f"{_STORED_LITELLM_PARAMS_SQL} -> 'complexity_router_config'"
394_CAPABILITY_DB_ROWS_SQL: Final[Mapping[str, str]] = MappingProxyType(
395 {
396 capability.key: f"""
397SELECT {_STORED_LITELLM_PARAMS_SQL} ->> 'model' AS model
398FROM "LiteLLM_ProxyModelTable"
399WHERE model_id <> $1
400 AND ({capability.sql_config_predicate.format(config=_STORED_COMPLEXITY_CONFIG_SQL)})
401"""
402 for capability in GATED_AUTO_ROUTER_CAPABILITIES
403 }
404)
407def _effective_complexity_router_config(
408 incoming_params: GenericLiteLLMParams | None, existing_params: GenericLiteLLMParams | None
409) -> object:
410 incoming: Final = None if incoming_params is None else incoming_params.complexity_router_config
411 existing: Final = None if existing_params is None else existing_params.complexity_router_config
412 if incoming is None:
413 return existing
414 if existing is None or incoming.get("classifier_type") != "jev" or existing.get("classifier_type") != "jev": 414 ↛ 416line 414 didn't jump to line 416 because the condition on line 414 was always true
415 return incoming
416 incoming_jev: Final[object] = incoming.get("jev_classifier_config")
417 existing_jev: Final[object] = existing.get("jev_classifier_config")
418 if not isinstance(incoming_jev, Mapping) or not isinstance(existing_jev, Mapping):
419 return incoming
420 supplied: Final = TypeAdapter(dict[str, object]).validate_python(incoming_jev)
421 stored: Final = TypeAdapter(dict[str, object]).validate_python(existing_jev)
422 same_base: Final = "api_base" not in supplied or supplied["api_base"] == stored.get("api_base")
423 transport: Final = MappingProxyType(
424 {
425 key: value
426 for key, value in stored.items()
427 if key in ("api_key", "api_base") and (key != "api_key" or same_base)
428 }
429 )
430 return { # mutable-ok: persisted JSON requires concrete nested dicts
431 **incoming,
432 "jev_classifier_config": { # mutable-ok: json.dumps cannot serialize MappingProxyType
433 **transport,
434 **supplied,
435 },
436 }
439def _effective_model(
440 incoming_params: GenericLiteLLMParams | None, existing_params: GenericLiteLLMParams | None
441) -> str | None:
442 """The model a write leaves on the row, decrypting an existing value only when the patch omits it."""
443 incoming: Final = None if incoming_params is None else incoming_params.model
444 if incoming is not None: 444 ↛ 446line 444 didn't jump to line 446 because the condition on line 444 was always true
445 return incoming
446 existing: Final = None if existing_params is None else existing_params.model
447 if existing is None:
448 return None
449 decrypted: Final = decrypt_value_helper(
450 value=existing,
451 key="model",
452 exception_type="debug",
453 return_original_value=True,
454 )
455 return decrypted if isinstance(decrypted, str) else None
458def _effective_complexity_router_params(
459 incoming_params: GenericLiteLLMParams | None, existing_params: GenericLiteLLMParams | None
460) -> Mapping[str, object]:
461 """The model and complexity config a write leaves, for placement and capability decisions."""
462 return MappingProxyType(
463 {
464 "model": _effective_model(incoming_params, existing_params),
465 "complexity_router_config": _effective_complexity_router_config(incoming_params, existing_params),
466 }
467 )
470def _member_auto_router_marker_for_update(
471 *,
472 incoming_params: updateLiteLLMParams | None,
473 existing: Deployment,
474 member_write: MemberAutoRouterWrite | None,
475) -> bool | None:
476 if member_write is not None: 476 ↛ 477line 476 didn't jump to line 477 because the condition on line 476 was never true
477 return True
478 if not existing.model_info.member_auto_router: 478 ↛ 480line 478 didn't jump to line 480 because the condition on line 478 was always true
479 return None
480 if incoming_params is None:
481 return True
482 if any(getattr(incoming_params, field, None) is not None for field in STRATEGY_ROUTER_PARAM_FIELDS):
483 return False
484 if incoming_params.model is not None and incoming_params.model != _effective_model(None, existing.litellm_params):
485 return False
486 return True
489def _decrypted_model(stored_model: object) -> str | None:
490 if not isinstance(stored_model, str):
491 return None
492 decrypted: Final = decrypt_value_helper(
493 value=stored_model, key="model", exception_type="debug", return_original_value=True
494 )
495 return decrypted if isinstance(decrypted, str) else None
498def _tuning_candidate(effective_params: Mapping[str, object], model_id: str | None) -> Mapping[str, object]:
499 return MappingProxyType(
500 {
501 "litellm_params": effective_params,
502 "model_info": MappingProxyType({"id": model_id, "db_model": True}),
503 }
504 )
507def _raise_on_tuning_quota_violation(
508 *,
509 candidate: Mapping[str, object],
510 others: Sequence[Mapping[str, object]],
511 baselines: Mapping[str, str],
512 limit: int | None,
513) -> None:
514 violation: Final = tuning_quota_violation(candidate=candidate, others=others, baselines=baselines, limit=limit)
515 if violation is not None:
516 raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}")
519@asynccontextmanager
520async def _auto_router_capability_slot(
521 prisma_client: PrismaClient,
522 *,
523 effective_params: Mapping[str, object],
524 model_id: str | None,
525 member_write: MemberAutoRouterWrite | None = None,
526) -> AsyncGenerator[_ProxyModelTable, None]:
527 """Hand out the model table to write through while the row's claim on a licensed capability is settled.
529 A write that leaves the row claiming a licensed capability under a limited license runs
530 inside one transaction that takes an advisory lock in its own statement before counting
531 (a statement's snapshot predates anything it locks), so pods cannot both pass the count:
532 the DB rows (any pod, either JSON shape) plus this proxy's config.yaml routers are judged
533 against the license limit and the write is refused with a 403 before it happens. The row
534 being edited keeps its own slot through ``model_id``. Member writes also recheck their
535 authorization under this lock. Team model bookkeeping needs a second connection and
536 must wait until the transaction has committed and the lock is released. The transaction
537 writes bypass the repository's publish-on-write, so the config change is published once
538 after commit, the way delete_team_models does.
540 A heuristic-v1 router whose tuning has moved off its recorded baseline is judged the same
541 way under the same lock, against the DB rows plus this proxy's config.yaml routers.
542 """
543 from litellm.proxy.proxy_server import (
544 _license_check, # pyright: ignore[reportPrivateUsage] # existing capability slot reads the proxy license singleton
545 heuristic_v1_tuning_baselines,
546 llm_router,
547 premium_user,
548 )
550 limit: Final = _license_check.auto_router_capability_limit()
551 capability: Final = gated_capability_of(effective_params)
552 baselines: Final = heuristic_v1_tuning_baselines
553 tuning_candidate: Final = _tuning_candidate(effective_params, model_id=model_id)
554 judges_tuning: Final = baselines is not None and is_mutable_tuned_candidate(tuning_candidate, baselines)
555 if member_write is None and (limit is None or (capability is None and not judges_tuning)): 555 ↛ 558line 555 didn't jump to line 558 because the condition on line 555 was always true
556 yield _proxy_model_table(prisma_client)
557 return
558 transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db)
559 transaction: Final = (
560 transaction_client.tx(timeout=datetime.timedelta(seconds=30))
561 if member_write is not None
562 else transaction_client.tx()
563 )
564 async with transaction as tx_ctx:
565 tables: Final[_TxModelTables] = tx_ctx
566 await tx_ctx.query_raw(_CAPABILITY_LOCK_SQL, AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY)
567 config_rows: Final = () if llm_router is None else tuple(llm_router.config_deployments())
568 if member_write is not None:
569 if member_write.model_id is not None:
570 await tx_ctx.query_raw(
571 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = $1 FOR UPDATE',
572 member_write.model_id,
573 )
574 pinned_client: Final = _TransactionClient(tx_ctx)
575 team_where: Final[prisma_types.LiteLLM_TeamTableWhereUniqueInput] = {"team_id": member_write.team_id}
576 team_include: Final[prisma_types.LiteLLM_TeamTableInclude] = {"litellm_model_table": True}
577 team_row: Final = await TeamRepository(pinned_client).table.find_unique(
578 where=team_where, include=team_include
579 )
580 if team_row is None or llm_router is None:
581 raise HTTPException(status_code=403, detail="The auto router's team or model catalog is unavailable.")
582 team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
583 authorize_member_auto_router_team(
584 user_api_key_dict=member_write.actor, team=team, premium_user=premium_user
585 )
586 if member_write.model_id is not None:
587 model_where: Final[prisma_types.LiteLLM_ProxyModelTableWhereInput] = {"model_id": member_write.model_id}
588 current_row: Final = await tables.litellm_proxymodeltable.find_unique(where=model_where)
589 current_identity: Final = (
590 StoredAutoRouterIdentity.model_validate(current_row.model_dump())
591 if current_row is not None
592 else None
593 )
594 current_model: Final = (
595 Deployment.model_validate(current_row.model_dump()) if current_row is not None else None
596 )
597 if (
598 current_identity is None
599 or current_identity.created_by != member_write.actor.user_id
600 or current_model is None
601 or current_model.model_info.team_id != member_write.team_id
602 ):
603 raise HTTPException(status_code=403, detail="Team members can update only their own auto routers.")
604 if current_identity.updated_at != member_write.updated_at:
605 raise HTTPException(status_code=409, detail="This auto router changed. Reload it before updating.")
606 else:
607 all_models: Final[prisma_types.LiteLLM_ProxyModelTableWhereInput] = {}
608 rows_for_names: Final = await tables.litellm_proxymodeltable.find_many(where=all_models)
609 stored_names: Final = tuple(
610 (
611 row.model_name,
612 model_info_as_mapping(row.model_info),
613 )
614 for row in rows_for_names
615 )
616 config_names: Final = tuple(
617 (str(row.get("model_name", "")), model_info_as_mapping(row.get("model_info")))
618 for row in config_rows
619 )
620 team_aliases: Final = team_model_aliases(team)
621 aliases: Final = (
622 *(llm_router.model_group_alias or ()),
623 *(litellm.model_alias_map or ()),
624 *(team_aliases or ()),
625 )
626 if member_write.public_name in aliases or any(
627 fnmatchcase(
628 member_write.public_name,
629 str(info.get("team_public_model_name") or name)
630 if info is not None and info.get("team_id") == member_write.team_id
631 else name,
632 )
633 for name, info in (*stored_names, *config_names)
634 if info is None or info.get("team_id") in (None, member_write.team_id)
635 ):
636 raise HTTPException(status_code=409, detail="This auto-router name is already used by a model.")
637 await authorize_member_auto_router_dependencies(
638 config=member_write.config,
639 default_model=member_write.default_model,
640 user_api_key_dict=member_write.actor,
641 team=team,
642 prisma_client=pinned_client,
643 llm_router=llm_router,
644 )
645 if capability is not None:
646 rows: Sequence[Mapping[str, object]] = await tx_ctx.query_raw(
647 _CAPABILITY_DB_ROWS_SQL[capability.key], model_id or ""
648 )
649 db_held: Final = sum(1 for row in rows if is_complexity_router_model(_decrypted_model(row.get("model"))))
650 held: Final = db_held + count_capability_routers(config_rows, capability=capability)
651 violation: Final = capability_limit_violation(capability=capability, held=held + 1, limit=limit)
652 if violation is not None:
653 raise HTTPException(
654 status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}"
655 )
656 if judges_tuning and baselines is not None:
657 model_rows: Final = await ModelRepository(_TransactionClient(tx_ctx)).find_all_except(model_id or "")
658 _raise_on_tuning_quota_violation(
659 candidate=tuning_candidate,
660 others=tuple(
661 MappingProxyType(
662 {
663 "litellm_params": row.litellm_params,
664 "model_info": MappingProxyType({"id": row.model_id, "db_model": True}),
665 }
666 )
667 for row in model_rows
668 )
669 + config_rows,
670 baselines=baselines,
671 limit=limit,
672 )
673 yield tables.litellm_proxymodeltable
674 await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable")
677ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add"
680def _raise_if_rate_limits_required_but_missing(*, litellm_params: GenericLiteLLMParams, enforced: bool) -> None:
681 """Require both rpm and tpm (each a positive value) when the operator opts in via config.yaml.
683 Off by default, so deployments keep adding models without limits. When
684 ``enforce_rpm_tpm_on_model_add: true`` is set under general_settings, a model added
685 without both rpm and tpm set to a positive value is rejected rather than stored
686 unbounded (or effectively excluded from routing by a zero/negative limit).
687 """
688 if not enforced: 688 ↛ 690line 688 didn't jump to line 690 because the condition on line 688 was always true
689 return
690 missing: Final = tuple(
691 field
692 for field, value in (("rpm", litellm_params.rpm), ("tpm", litellm_params.tpm))
693 if value is None or value <= 0
694 )
695 if not missing:
696 return
697 raise ProxyException(
698 message=(
699 f"{' and '.join(missing)} must be set to a positive value when "
700 f"'{ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING}' is enabled in general_settings"
701 ),
702 type=ProxyErrorTypes.validation_error.value,
703 code=status.HTTP_400_BAD_REQUEST,
704 param=f"litellm_params.{missing[0]}",
705 )
708_PTU_PRICED_PAIR: Final = frozenset({"ptu_count", "cost_per_ptu_per_hour"})
711def _explicitly_cleared_ptu_fields(model_info: ModelInfo | None) -> frozenset[str]:
712 """The PTU fields a patch sends as an explicit null, which update_db_model drops.
714 Empty while the feature is off, so disabling pauses PTU rather than letting a client
715 that round-trips a model_info blob erase a configuration set up during an earlier opt-in.
716 """
717 if model_info is None or not is_ptu_cost_attribution_enabled():
718 return frozenset()
719 return frozenset(
720 field
721 for field in PTU_MODEL_INFO_FIELDS
722 if field in model_info.model_fields_set and getattr(model_info, field) is None
723 )
726def _merged_ptu_model_info(*, db_model: Deployment, patch_data: updateDeployment) -> Mapping[str, object]:
727 """The model_info a patch would store, which is the stored blob updated by the patch.
729 A PTU invariant holds over the deployment as it will exist, not over whichever subset
730 of fields a caller happened to send.
731 """
732 stored: Final = db_model.model_info.model_dump(exclude_none=True) if db_model.model_info else _EMPTY_MODEL_INFO
733 incoming: Final = (
734 patch_data.model_info.model_dump(exclude_none=True) if patch_data.model_info else _EMPTY_MODEL_INFO
735 )
736 cleared: Final = _explicitly_cleared_ptu_fields(patch_data.model_info)
737 return MappingProxyType({k: v for k, v in {**stored, **incoming}.items() if k not in cleared})
740def _raise_if_ptu_cost_attribution_disabled(incoming_model_info: Mapping[str, object]) -> None:
741 """Reject PTU model_info fields unless the operator opted into PTU cost attribution.
743 Takes the incoming request's model_info rather than the merged deployment, so an
744 unrelated patch of a model that still stores PTU config from an earlier opt-in is
745 left alone. The fields are rejected rather than dropped so a caller never believes
746 a flat cost was configured while the rollup that would price it is not running.
748 Only a value is rejected. An explicit null reaches the clear loop, which is gated on
749 the same flag, so a disabled proxy neither writes PTU config nor erases what an
750 earlier opt-in stored. Disabling pauses the feature rather than discarding its setup.
751 """
752 if is_ptu_cost_attribution_enabled(): 752 ↛ 753line 752 didn't jump to line 753 because the condition on line 752 was never true
753 return
754 supplied: Final = tuple(field for field in PTU_MODEL_INFO_FIELDS if incoming_model_info.get(field) is not None)
755 if not supplied:
756 return
757 raise HTTPException(
758 status_code=400,
759 detail=(
760 f"PTU cost attribution is disabled, so {', '.join(supplied)} cannot be set. "
761 f"Set {PTU_COST_ATTRIBUTION_ENV_VAR}=true to enable it."
762 ),
763 )
766def _validate_ptu_model_info(model_info: Mapping[str, object]) -> None:
767 """Enforce the PTU cross-field invariant on the effective model_info.
769 The rules live in litellm_core_utils.ptu_pricing so that config.yaml registration
770 refuses the same deployments this endpoint does, for the same reason. Per-field bounds
771 (positive count, non-negative rate) are enforced by ModelInfo itself.
773 Registration additionally requires an operator-declared ``model_info.id``, which this
774 endpoint does not: a stored deployment already holds a stable primary key, where a
775 config-declared one is otherwise keyed by a hash of its own parameters.
776 """
777 error: Final = ptu_config_error(model_info)
778 if error is not None: 778 ↛ 779line 778 didn't jump to line 779 because the condition on line 778 was never true
779 raise HTTPException(status_code=400, detail=error)
782# The mirrored per-token pricing fields plus the remaining rates the public cost map or a
783# provider default would otherwise supply (the cache back-fills, the Maps grounding rate). An
784# unset field falls back to those sources, so a field left out here is one a PTU deployment
785# still bills.
786# tiered_pricing is the one mirrored field that is a table of ranges, not a rate, so it is stored
787# empty (see _PTU_EMPTIED_PRICING_FIELDS): its tiers outrank the zeros written beside them, so
788# dropping it would leave the cost map's tiers billing the traffic the reserved capacity covers.
789_PTU_ZEROED_PRICING_FIELDS: Final = PTU_ZEROED_PRICING_FIELDS
790_PTU_EMPTIED_PRICING_FIELDS: Final = PTU_EMPTIED_PRICING_FIELDS
791_PTU_ZEROED_PRICING: Final[Mapping[str, float | tuple[()]]] = MappingProxyType(
792 {
793 **dict.fromkeys(_PTU_ZEROED_PRICING_FIELDS, 0.0),
794 **dict.fromkeys(_PTU_EMPTIED_PRICING_FIELDS, ()),
795 }
796)
797_NO_PRICING_OVERRIDE: Final[Mapping[str, float | tuple[()]]] = MappingProxyType({})
798_EMPTY_MODEL_INFO: Final[Mapping[str, object]] = _NO_PRICING_OVERRIDE
799# Rate fields only. CustomPricingLiteLLMParams also carries settings that are not charges
800# (an embedding's output_vector_size, the regional uplift multipliers), and zeroing one of
801# those would destroy the deployment's configuration rather than stop a charge.
802_CUSTOM_PRICING_FIELDS: Final = CUSTOM_PRICING_FIELDS
803# search_context_cost_per_query holds its rates in a table keyed by context size, and an absent
804# table means the provider's own default rate rather than free (litellm/llms/gemini/cost_calculator
805# falls back to $0.035), so it is zeroed in place rather than emptied like tiered_pricing, and
806# written on every PTU deployment rather than only where a table is already stored.
807_PTU_ZEROED_TABLE_FIELDS: Final = PTU_ZEROED_TABLE_FIELDS
808_SEARCH_CONTEXT_SIZES: Final = SEARCH_CONTEXT_SIZES
811def _is_nonzero_rate(value: object) -> bool:
812 return isinstance(value, (int, float)) and not isinstance(value, bool) and value != 0
815def _is_nonzero_price(value: object) -> bool:
816 if isinstance(value, dict): # an all-zero table is how a rate is expressed as free
817 return any(_is_nonzero_rate(rate) for rate in value.values())
818 return _is_nonzero_rate(value)
821def _is_zero_price(value: object) -> bool:
822 if isinstance(value, dict):
823 return bool(value) and not _is_nonzero_price(value)
824 if isinstance(value, (list, tuple)):
825 return not value
826 return isinstance(value, (int, float)) and not isinstance(value, bool) and value == 0
829def _raise_if_ptu_deployment_is_priced(*, model_info: Mapping[str, object], supplied: Mapping[str, object]) -> None:
830 """Refuse a rate the caller supplies for a deployment that bills reserved capacity.
832 Separate from the zeroing so the team-model path can run it before it touches the team, whose
833 ACL write autocommits: a refusal raised after it would leave the team changed and the
834 deployment row never written.
835 """
836 if not is_ptu_cost_attribution_enabled():
837 return
838 if model_info.get("ptu_count") is None or model_info.get("cost_per_ptu_per_hour") is None:
839 return
840 priced: Final = tuple(
841 sorted(
842 tuple(field for field in _CUSTOM_PRICING_FIELDS if _is_nonzero_price(supplied.get(field)))
843 + tuple(field for field in _PTU_EMPTIED_PRICING_FIELDS if supplied.get(field))
844 )
845 )
846 if not priced:
847 return
848 raise HTTPException(
849 status_code=400,
850 detail=(
851 f"A PTU deployment bills by reserved capacity, so {', '.join(priced)} cannot be charged on "
852 "top of it. Send 0 or no value, or remove ptu_count and cost_per_ptu_per_hour to bill per token."
853 ),
854 )
857def _ptu_zeroed_pricing(
858 *,
859 model_info: Mapping[str, object],
860 litellm_params: Mapping[str, object],
861 supplied: Mapping[str, object],
862) -> Mapping[str, float | tuple[()] | Mapping[str, float]]:
863 """The pricing a PTU deployment must carry, empty unless one is being stored.
865 Reserved capacity is already billed by the flat cost the rollup writes, so charging the
866 traffic it serves bills the same tokens twice. Left unset the rate falls back to the public
867 cost map, which makes the double charge the default rather than an opt-in.
869 Only a price the caller supplies is refused. A non-zero price already on the row is zeroed
870 instead, so a deployment priced through a path this rule does not cover heals on its next
871 save rather than rejecting every later edit of a field that has nothing to do with pricing.
873 ``supplied`` is the caller's litellm_params alone, because that is the blob a price is
874 authored on. model_info's copy is written by the server, both by the mirror in
875 ``Deployment.__init__`` and by the cost-map defaults /model/info fills in, so a client that
876 round-trips a model_info blob sends back prices it never chose.
877 """
878 if not is_ptu_cost_attribution_enabled(): 878 ↛ 880line 878 didn't jump to line 880 because the condition on line 878 was always true
879 return _NO_PRICING_OVERRIDE
880 if model_info.get("ptu_count") is None or model_info.get("cost_per_ptu_per_hour") is None:
881 return _NO_PRICING_OVERRIDE
882 _raise_if_ptu_deployment_is_priced(model_info=model_info, supplied=supplied)
883 stored: Final = frozenset(
884 field
885 for field in _CUSTOM_PRICING_FIELDS
886 if _is_nonzero_price(model_info.get(field)) or _is_nonzero_price(litellm_params.get(field))
887 )
888 return MappingProxyType(
889 {
890 **_PTU_ZEROED_PRICING,
891 **dict.fromkeys(_PTU_ZEROED_TABLE_FIELDS, dict.fromkeys(_SEARCH_CONTEXT_SIZES, 0.0)),
892 **dict.fromkeys(stored - _PTU_ZEROED_TABLE_FIELDS, 0.0),
893 }
894 )
897def _ptu_pricing_delta(
898 *,
899 stored_model_info: Mapping[str, object],
900 model_info: Mapping[str, object],
901 litellm_params: Mapping[str, object],
902 patch: updateDeployment,
903) -> tuple[Mapping[str, float | tuple[()] | Mapping[str, float]], frozenset[str]]:
904 """The pricing a patch must write into both blobs, and the pricing it must drop from them.
906 A patch that takes the deployment off PTU takes the zeroed pricing with it, since the zeros
907 exist only to stop the double charge. Left behind they would serve the deployment for free.
908 Reading the stored row rather than the patch alone keeps that release off a deployment that
909 never carried PTU config, whose zero price is a rate its operator chose. A zero the patch
910 itself carries is released with the rest, because the dashboard echoes the whole stored
911 blob on every save, so a supplied zero cannot be told apart from the one this rule wrote.
913 The release spans every field the zeroing could have written, not just the mirrored ones, or
914 a rate zeroed on the way in (per-second, per-character tiers) would bill nothing forever.
915 """
916 supplied: Final = patch.litellm_params.model_dump(exclude_none=True) if patch.litellm_params else _EMPTY_MODEL_INFO
917 zeroed: Final = _ptu_zeroed_pricing(model_info=model_info, litellm_params=litellm_params, supplied=supplied)
918 if zeroed:
919 return zeroed, frozenset()
920 was_ptu: Final = any(stored_model_info.get(field) is not None for field in _PTU_PRICED_PAIR)
921 if not was_ptu or not _explicitly_cleared_ptu_fields(patch.model_info) & _PTU_PRICED_PAIR:
922 return _NO_PRICING_OVERRIDE, frozenset()
923 return _NO_PRICING_OVERRIDE, frozenset(
924 field
925 for field in _CUSTOM_PRICING_FIELDS.union(_PTU_ZEROED_PRICING_FIELDS, _PTU_EMPTIED_PRICING_FIELDS)
926 if _is_zero_price(model_info.get(field)) or _is_zero_price(litellm_params.get(field))
927 )
930def _ptu_priced_deployment(model_params: Deployment) -> Deployment:
931 """``model_params`` with PTU pricing applied, or itself when it configures no PTU."""
932 model_info: Final = model_params.model_info.model_dump(exclude_none=True)
933 litellm_params: Final = model_params.litellm_params.model_dump(exclude_none=True)
934 override: Final = _ptu_zeroed_pricing(model_info=model_info, litellm_params=litellm_params, supplied=litellm_params)
935 if not override: 935 ↛ 939line 935 didn't jump to line 939 because the condition on line 935 was always true
936 return model_params
937 # model_copy validates nothing, so the emptied tier table has to arrive as the list the field
938 # declares or Pydantic warns on every later dump of it
939 stored: Final = MappingProxyType(
940 {key: [] if isinstance(value, tuple) else value for key, value in override.items()}
941 )
942 return model_params.model_copy(
943 update=MappingProxyType(
944 {
945 "litellm_params": model_params.litellm_params.model_copy(update=stored),
946 "model_info": model_params.model_info.model_copy(update=stored),
947 }
948 )
949 )
952def _cost_map_entry(db_model: Deployment, incoming_model_info: Mapping[str, object]) -> Mapping[str, object]:
953 base_model: Final = incoming_model_info.get("base_model")
954 lookup: Final = base_model if isinstance(base_model, str) else _decrypted_model(db_model.litellm_params.model)
955 if lookup is None:
956 return MappingProxyType({})
957 with suppress(Exception):
958 return MappingProxyType(dict(litellm.get_model_info(model=lookup)))
959 return MappingProxyType({})
962LoadedCatalog: TypeAlias = Callable[[], Mapping[str, Mapping[str, object]]] # mutable-ok: Callable parameter syntax
965def _loaded_catalog_entry(
966 incoming_model_info: Mapping[str, object], loaded_catalog: LoadedCatalog
967) -> Mapping[str, object]:
968 catalog_key: Final = incoming_model_info.get(COST_MAP_LOOKUP_KEY)
969 if not isinstance(catalog_key, str):
970 return MappingProxyType({})
971 return loaded_catalog().get(catalog_key, MappingProxyType({}))
974def update_db_model(
975 db_model: Deployment,
976 updated_patch: updateDeployment,
977 loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map,
978) -> PrismaCompatibleUpdateDBModel:
979 if updated_patch.model_info is not None:
980 _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True))
981 merged_model_name: Final = updated_patch.model_name or db_model.model_name
982 merged_litellm_params: Final = db_model.litellm_params.model_dump(exclude_none=True)
983 stored_model_info: Final = db_model.model_info.model_dump(exclude_none=True)
984 echoed_pricing: Final = echoed_cost_map_pricing_fields(stored_model_info)
985 merged_model_info: Final[dict[str, object]] = {
986 k: v for k, v in stored_model_info.items() if k not in echoed_pricing
987 }
989 # update litellm params
990 if updated_patch.litellm_params:
991 # Encrypt any sensitive values
992 encrypted_params: Final = {
993 k: (
994 _effective_complexity_router_config(updated_patch.litellm_params, db_model.litellm_params)
995 if k == "complexity_router_config"
996 else encrypt_value_helper(v)
997 )
998 for k, v in updated_patch.litellm_params.model_dump(exclude_none=True).items()
999 }
1001 merged_litellm_params.update(encrypted_params)
1003 # update model info
1004 if updated_patch.model_info:
1005 incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True)
1006 echoed_fields: Final = echoed_cost_map_fields(
1007 incoming_model_info,
1008 _cost_map_entry(db_model, incoming_model_info),
1009 _loaded_catalog_entry(incoming_model_info, loaded_catalog),
1010 )
1011 merged_model_info.update(
1012 MappingProxyType(
1013 dict(
1014 (k, v)
1015 for k, v in without_server_derived_pricing(incoming_model_info).items()
1016 if k not in echoed_fields
1017 )
1018 )
1019 )
1020 for k in echoed_fields:
1021 if k in merged_model_info and merged_model_info[k] != incoming_model_info[k]:
1022 del merged_model_info[k]
1024 # Honor explicit-null clears LAST, after both merges, so a model_info blob a client
1025 # passes through cannot silently undo a litellm_params clear via .update().
1026 #
1027 # Restricted to SPECIAL_MODEL_INFO_PARAMS (input/output cost per token/character
1028 # and cache read/write costs) so this path cannot be used to null out privileged
1029 # model_info fields like team_id or access groups. SPECIAL_MODEL_INFO_PARAMS are
1030 # mirrored between litellm_params and model_info by Deployment.__init__, so the
1031 # clear propagates to both blobs.
1032 if updated_patch.litellm_params:
1033 for field in updated_patch.litellm_params.model_fields_set:
1034 if getattr(updated_patch.litellm_params, field) is None and field in NULL_CLEARABLE_LITELLM_PARAMS:
1035 merged_litellm_params.pop(field, None)
1036 merged_model_info.pop(field, None)
1037 elif (
1038 field
1039 in (
1040 "auto_router_routing_compression",
1041 "auto_router_model_compression",
1042 )
1043 and getattr(updated_patch.litellm_params, field) is None
1044 ):
1045 merged_litellm_params.pop(field, None)
1046 if updated_patch.model_info:
1047 for field in updated_patch.model_info.model_fields_set:
1048 if field in SPECIAL_MODEL_INFO_PARAMS and getattr(updated_patch.model_info, field) is None:
1049 merged_model_info.pop(field, None)
1050 merged_litellm_params.pop(field, None)
1051 for field in _explicitly_cleared_ptu_fields(updated_patch.model_info):
1052 merged_model_info.pop(field, None)
1054 _validate_ptu_model_info(merged_model_info)
1055 ptu_pricing, ptu_released = _ptu_pricing_delta(
1056 stored_model_info=db_model.model_info.model_dump(exclude_none=True)
1057 if db_model.model_info
1058 else _EMPTY_MODEL_INFO,
1059 model_info=merged_model_info,
1060 litellm_params=merged_litellm_params,
1061 patch=updated_patch,
1062 )
1063 merged_model_info.update(ptu_pricing)
1064 merged_litellm_params.update(ptu_pricing)
1065 for field in ptu_released:
1066 merged_model_info.pop(field, None)
1067 merged_litellm_params.pop(field, None)
1069 # convert to prisma compatible format
1071 for key, value in merged_model_info.items():
1072 if isinstance(value, datetime.datetime):
1073 merged_model_info[key] = value.isoformat()
1075 prisma_compatible_model_dict: Final = PrismaCompatibleUpdateDBModel(
1076 model_name=merged_model_name,
1077 litellm_params=json.dumps(merged_litellm_params),
1078 model_info=json.dumps(merged_model_info),
1079 )
1081 if updated_patch.blocked is not None:
1082 prisma_compatible_model_dict["blocked"] = updated_patch.blocked
1084 return prisma_compatible_model_dict
1087@router.patch(
1088 "/model/{model_id}/update",
1089 tags=["model management"],
1090 dependencies=[Depends(user_api_key_auth)],
1091)
1092async def patch_model(
1093 model_id: str, # Get model_id from path parameter
1094 patch_data: updateDeployment, # Create a specific schema for PATCH operations
1095 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
1096):
1097 """
1098 PATCH Endpoint for partial model updates.
1100 Only updates the fields specified in the request while preserving other existing values.
1101 Follows proper PATCH semantics by only modifying provided fields.
1103 Args:
1104 model_id: The ID of the model to update
1105 patch_data: The fields to update and their new values
1106 user_api_key_dict: User authentication information
1108 Returns:
1109 Updated model information
1111 Raises:
1112 ProxyException: For various error conditions including authentication and database errors
1113 """
1114 from litellm.proxy.proxy_server import (
1115 litellm_proxy_admin_name,
1116 llm_router,
1117 premium_user,
1118 prisma_client,
1119 store_model_in_db,
1120 user_api_key_cache,
1121 )
1123 try:
1124 if prisma_client is None: 1124 ↛ 1125line 1124 didn't jump to line 1125 because the condition on line 1124 was never true
1125 raise HTTPException(
1126 status_code=500,
1127 detail={"error": CommonProxyErrors.db_not_connected_error.value},
1128 )
1130 # Verify model exists and is stored in DB
1131 if not store_model_in_db: 1131 ↛ 1132line 1131 didn't jump to line 1132 because the condition on line 1131 was never true
1132 raise ProxyException(
1133 message="Model updates only supported for DB-stored models",
1134 type=ProxyErrorTypes.validation_error.value,
1135 code=status.HTTP_400_BAD_REQUEST,
1136 param=None,
1137 )
1139 # Fetch existing model
1140 db_model: Final = await get_db_model(model_id=model_id, prisma_client=prisma_client)
1142 if db_model is None: 1142 ↛ 1158line 1142 didn't jump to line 1158 because the condition on line 1142 was always true
1143 # Check if model exists in config but not DB
1144 if llm_router and llm_router.get_deployment(model_id=model_id) is not None: 1144 ↛ 1145line 1144 didn't jump to line 1145 because the condition on line 1144 was never true
1145 raise ProxyException(
1146 message="Cannot edit config-based model. Store model in DB via /model/new first.",
1147 type=ProxyErrorTypes.validation_error.value,
1148 code=status.HTTP_400_BAD_REQUEST,
1149 param=None,
1150 )
1151 raise ProxyException(
1152 message=f"Model {model_id} not found on proxy.",
1153 type=ProxyErrorTypes.not_found_error,
1154 code=status.HTTP_404_NOT_FOUND,
1155 param=None,
1156 )
1158 write_authorization: Final = await ModelManagementAuthChecks.can_user_make_model_call(
1159 model_params=db_model,
1160 user_api_key_dict=user_api_key_dict,
1161 prisma_client=prisma_client,
1162 premium_user=premium_user,
1163 member_operation="update",
1164 incoming_model_params=patch_data,
1165 )
1166 member_write: Final = write_authorization if isinstance(write_authorization, MemberAutoRouterWrite) else None
1167 member_marker: Final = _member_auto_router_marker_for_update(
1168 incoming_params=patch_data.litellm_params, existing=db_model, member_write=member_write
1169 )
1170 marker_info: Final = (
1171 ModelInfo(id=db_model.model_info.id)
1172 if member_write is not None
1173 else patch_data.model_info or ModelInfo(id=db_model.model_info.id)
1174 )
1175 effective_info: Final = (
1176 marker_info.model_copy(update=MappingProxyType({"member_auto_router": member_marker}))
1177 if member_marker is not None
1178 else patch_data.model_info
1179 )
1180 effective_patch: Final = (
1181 patch_data.model_copy(
1182 update=MappingProxyType(
1183 {
1184 "model_name": None if member_write is not None else patch_data.model_name,
1185 "model_info": effective_info,
1186 }
1187 )
1188 )
1189 if member_marker is not None
1190 else patch_data
1191 )
1193 # Pause/resume (`blocked`) is a proxy-admin-only privilege. Team admins
1194 # passed the auth check above for team-scoped models, but they must not
1195 # be able to unblock (or block) a model their proxy admin has paused.
1196 if patch_data.blocked is not None and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
1197 raise ProxyException(
1198 message="Only proxy admins can change a model's blocked flag.",
1199 type=ProxyErrorTypes.auth_error.value,
1200 code=status.HTTP_403_FORBIDDEN,
1201 param="blocked",
1202 )
1204 ModelManagementAuthChecks.can_user_attach_credential(
1205 litellm_params=patch_data.litellm_params,
1206 user_api_key_dict=user_api_key_dict,
1207 existing_litellm_params=db_model.litellm_params,
1208 null_detaches=True,
1209 )
1210 await _raise_on_invalid_credential_name(patch_data.litellm_params, db_model.litellm_params, prisma_client)
1212 ModelManagementAuthChecks.can_user_set_aws_session_tags(
1213 litellm_params=patch_data.litellm_params,
1214 user_api_key_dict=user_api_key_dict,
1215 existing_litellm_params=db_model.litellm_params,
1216 )
1218 _raise_on_strategy_router_write_violation(
1219 incoming_params=patch_data.litellm_params,
1220 existing_params=db_model.litellm_params,
1221 )
1223 effective_params: Final = _effective_complexity_router_params(
1224 patch_data.litellm_params, db_model.litellm_params
1225 )
1226 requested_model_name: Final = patch_data.model_name
1227 stored_model_name: str | None = None
1229 async def write_row(update_data: PrismaCompatibleUpdateDBModel) -> _ProxyModelRow | None:
1230 nonlocal stored_model_name
1231 stored_model_name = update_data.get("model_name")
1232 update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
1233 update_data["updated_at"] = cast(str, get_utc_datetime())
1234 async with _auto_router_capability_slot(
1235 prisma_client,
1236 effective_params=effective_params,
1237 model_id=model_id,
1238 member_write=member_write,
1239 ) as table:
1240 return await table.update(where={"model_id": model_id}, data=update_data)
1242 # Handle team model updates with proper alias management
1243 updated_model: Final = await _update_team_model_in_db(
1244 db_model=db_model,
1245 patch_data=effective_patch,
1246 user_api_key_dict=user_api_key_dict,
1247 prisma_client=prisma_client,
1248 write_row=write_row,
1249 )
1251 if updated_model is None:
1252 raise ProxyException(
1253 message=f"Model {model_id} not found on proxy.",
1254 type=ProxyErrorTypes.not_found_error,
1255 code=status.HTTP_404_NOT_FOUND,
1256 param=None,
1257 )
1259 if (
1260 stored_model_name is not None
1261 and stored_model_name == requested_model_name
1262 and stored_model_name != db_model.model_name
1263 ):
1264 await sync_access_groups_for_renamed_model(
1265 prisma_client=prisma_client,
1266 model_id=model_id,
1267 old_name=db_model.model_name,
1268 new_name=stored_model_name,
1269 llm_router=llm_router,
1270 )
1271 await sync_model_allowlists_for_renamed_model(
1272 prisma_client=prisma_client,
1273 model_id=model_id,
1274 old_name=db_model.model_name,
1275 new_name=stored_model_name,
1276 llm_router=llm_router,
1277 user_api_key_cache=user_api_key_cache,
1278 )
1280 # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates)
1281 live_before_reload: Final = live_model_ids_snapshot()
1282 reload_outcome: Final = await clear_cache()
1284 ## CREATE AUDIT LOG ##
1285 asyncio.create_task(
1286 create_object_audit_log(
1287 object_id=model_id,
1288 action="updated",
1289 user_api_key_dict=user_api_key_dict,
1290 table_name=LitellmTableNames.PROXY_MODEL_TABLE_NAME,
1291 before_value=db_model.model_dump_json(exclude_none=True),
1292 after_value=updated_model.model_dump_json(exclude_none=True),
1293 litellm_changed_by=user_api_key_dict.user_id,
1294 litellm_proxy_admin_name=LITELLM_PROXY_ADMIN_NAME,
1295 )
1296 )
1298 raise_if_reload_degraded_serving(
1299 before=live_before_reload,
1300 written_models=[(model_id, getattr(updated_model, "model_info", None))],
1301 action="update",
1302 still_desired=reload_outcome.still_desired,
1303 live_after=reload_outcome.live_after,
1304 )
1306 return updated_model
1308 except Exception as e:
1309 verbose_proxy_logger.exception("Error in patch_model: %s", e)
1311 if isinstance(e, (HTTPException, ProxyException)): 1311 ↛ 1314line 1311 didn't jump to line 1314 because the condition on line 1311 was always true
1312 raise e
1314 raise ProxyException(
1315 message=f"Error updating model: {e}",
1316 type=ProxyErrorTypes.internal_server_error,
1317 code=status.HTTP_500_INTERNAL_SERVER_ERROR,
1318 param=None,
1319 )
1322async def _set_model_blocked_status(
1323 data: BlockModelRequest,
1324 user_api_key_dict: UserAPIKeyAuth,
1325 blocked: bool,
1326 action: Literal["blocked", "unblocked"],
1327 litellm_changed_by: str | None,
1328) -> LiteLLM_ProxyModelTable | None:
1329 from litellm.proxy.proxy_server import (
1330 litellm_proxy_admin_name,
1331 llm_router,
1332 prisma_client,
1333 store_model_in_db,
1334 )
1336 try:
1337 if prisma_client is None: 1337 ↛ 1338line 1337 didn't jump to line 1338 because the condition on line 1337 was never true
1338 raise HTTPException(
1339 status_code=500,
1340 detail={"error": CommonProxyErrors.db_not_connected_error.value},
1341 )
1343 if store_model_in_db is not True: 1343 ↛ 1344line 1343 didn't jump to line 1344 because the condition on line 1343 was never true
1344 raise ProxyException(
1345 message="Model updates only supported for DB-stored models",
1346 type=ProxyErrorTypes.validation_error.value,
1347 code=status.HTTP_400_BAD_REQUEST,
1348 param=None,
1349 )
1351 if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: 1351 ↛ 1352line 1351 didn't jump to line 1352 because the condition on line 1351 was never true
1352 raise ProxyException(
1353 message="Only proxy admins can change a model's blocked flag.",
1354 type=ProxyErrorTypes.auth_error.value,
1355 code=status.HTTP_403_FORBIDDEN,
1356 param="blocked",
1357 )
1359 db_model: Final = await get_db_model(
1360 model_id=data.model_id,
1361 prisma_client=prisma_client,
1362 )
1364 if db_model is None: 1364 ↛ 1379line 1364 didn't jump to line 1379 because the condition on line 1364 was always true
1365 if llm_router and llm_router.get_deployment(model_id=data.model_id) is not None: 1365 ↛ 1366line 1365 didn't jump to line 1366 because the condition on line 1365 was never true
1366 raise ProxyException(
1367 message="Cannot edit config-based model. Store model in DB via /model/new first.",
1368 type=ProxyErrorTypes.validation_error.value,
1369 code=status.HTTP_400_BAD_REQUEST,
1370 param=None,
1371 )
1372 raise ProxyException(
1373 message=f"Model {data.model_id} not found on proxy.",
1374 type=ProxyErrorTypes.not_found_error,
1375 code=status.HTTP_404_NOT_FOUND,
1376 param=None,
1377 )
1379 updated_model: Final = await ModelRepository(prisma_client).table.update(
1380 where={"model_id": data.model_id},
1381 data={
1382 "blocked": blocked,
1383 "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
1384 "updated_at": cast(str, get_utc_datetime()),
1385 },
1386 )
1388 live_before_reload: Final = live_model_ids_snapshot()
1389 reload_outcome: Final = await clear_cache()
1391 asyncio.create_task(
1392 create_object_audit_log(
1393 object_id=data.model_id,
1394 action=action,
1395 user_api_key_dict=user_api_key_dict,
1396 table_name=LitellmTableNames.PROXY_MODEL_TABLE_NAME,
1397 before_value=db_model.model_dump_json(exclude_none=True),
1398 after_value=(
1399 updated_model.model_dump_json(exclude_none=True) if isinstance(updated_model, BaseModel) else None
1400 ),
1401 litellm_changed_by=litellm_changed_by,
1402 litellm_proxy_admin_name=litellm_proxy_admin_name,
1403 )
1404 )
1406 raise_if_reload_degraded_serving(
1407 before=live_before_reload,
1408 written_models=[(data.model_id, getattr(updated_model, "model_info", None))],
1409 action=action,
1410 still_desired=reload_outcome.still_desired,
1411 live_after=reload_outcome.live_after,
1412 )
1414 return updated_model # pyright: ignore[reportReturnType] # prisma row, coerced by this route's response_model
1416 except Exception as e:
1417 verbose_proxy_logger.exception("Error in model %s: %s", action, e)
1419 if isinstance(e, (HTTPException, ProxyException)):
1420 raise e
1422 raise ProxyException(
1423 message=f"Error updating model blocked status: {e}",
1424 type=ProxyErrorTypes.internal_server_error,
1425 code=status.HTTP_500_INTERNAL_SERVER_ERROR,
1426 param=None,
1427 )
1430@router.post(
1431 "/model/block",
1432 tags=["model management"],
1433 dependencies=[Depends(user_api_key_auth)],
1434)
1435async def block_model(
1436 data: BlockModelRequest,
1437 http_request: Request,
1438 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
1439 litellm_changed_by: str | None = Header(
1440 None,
1441 description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability",
1442 ),
1443) -> LiteLLM_ProxyModelTable | None:
1444 """
1445 Block a DB-stored model deployment from serving requests.
1447 Parameters:
1448 - model_id: str - The model deployment id to block.
1449 """
1450 return await _set_model_blocked_status(
1451 data=data,
1452 user_api_key_dict=user_api_key_dict,
1453 blocked=True,
1454 action="blocked",
1455 litellm_changed_by=litellm_changed_by,
1456 )
1459@router.post(
1460 "/model/unblock",
1461 tags=["model management"],
1462 dependencies=[Depends(user_api_key_auth)],
1463)
1464async def unblock_model(
1465 data: BlockModelRequest,
1466 http_request: Request,
1467 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
1468 litellm_changed_by: str | None = Header(
1469 None,
1470 description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability",
1471 ),
1472) -> LiteLLM_ProxyModelTable | None:
1473 """
1474 Unblock a DB-stored model deployment so it can serve requests again.
1476 Parameters:
1477 - model_id: str - The model deployment id to unblock.
1478 """
1479 return await _set_model_blocked_status(
1480 data=data,
1481 user_api_key_dict=user_api_key_dict,
1482 blocked=False,
1483 action="unblocked",
1484 litellm_changed_by=litellm_changed_by,
1485 )
1488################################# Helper Functions #################################
1489####################################################################################
1490####################################################################################
1491####################################################################################
1494async def _add_model_to_db(
1495 model_params: Deployment,
1496 user_api_key_dict: UserAPIKeyAuth,
1497 prisma_client: PrismaClient,
1498 new_encryption_key: str | None = None,
1499 should_create_model_in_db: bool = True,
1500 slot: AbstractAsyncContextManager[_ProxyModelTable] | None = None,
1501) -> "_ProxyModelRow | LiteLLM_ProxyModelTable":
1502 # encrypt litellm params #
1503 _litellm_params_dict: Final = model_params.litellm_params.dict(exclude_none=True)
1504 _original_litellm_model_name: Final = model_params.litellm_params.model
1505 for k, v in _litellm_params_dict.items():
1506 encrypted_value = encrypt_value_helper(value=v, new_encryption_key=new_encryption_key)
1507 model_params.litellm_params[k] = encrypted_value
1508 _data: Final[dict] = {
1509 "model_id": model_params.model_info.id,
1510 "model_name": model_params.model_name,
1511 "litellm_params": model_params.litellm_params.model_dump_json(exclude_none=True),
1512 "model_info": model_params.model_info.model_dump_json(exclude_none=True),
1513 "created_by": user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME,
1514 "updated_by": user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME,
1515 }
1516 if model_params.model_info.id is not None: 1516 ↛ 1518line 1516 didn't jump to line 1518 because the condition on line 1516 was always true
1517 _data["model_id"] = model_params.model_info.id
1518 _create_data: Final = cast("Mapping[str, object]", _data) # cast-ok: str-keyed json payload built just above
1519 if not should_create_model_in_db: 1519 ↛ 1520line 1519 didn't jump to line 1520 because the condition on line 1519 was never true
1520 return LiteLLM_ProxyModelTable(**_data)
1521 if slot is None: 1521 ↛ 1522line 1521 didn't jump to line 1522 because the condition on line 1521 was never true
1522 return await _proxy_model_table(prisma_client).create(data=_create_data)
1523 async with slot as table:
1524 return await table.create(data=_create_data)
1527async def _add_team_model_to_db(
1528 model_params: Deployment,
1529 user_api_key_dict: UserAPIKeyAuth,
1530 prisma_client: PrismaClient,
1531 slot: AbstractAsyncContextManager[_ProxyModelTable] | None = None,
1532) -> "_ProxyModelRow | LiteLLM_ProxyModelTable | None":
1533 """
1534 If 'team_id' is provided,
1536 - generate a unique 'model_name' for the model (e.g. 'model_name_{team_id}_{uuid})
1537 - store the model in the db with the unique 'model_name'
1538 - add the public model name to the team's allowed models list
1539 """
1540 from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache
1542 _team_id: Final = model_params.model_info.team_id
1543 if _team_id is None:
1544 return None
1546 # Capture the original public name FIRST, before any mutations
1547 original_model_name: Final = model_params.model_name
1549 # Set team_public_model_name in model_info using the captured original_model_name
1550 # This must happen BEFORE mutating model_params.model_name so _add_model_to_db
1551 # serializes the correct team_public_model_name (not the internal UUID name)
1552 if original_model_name:
1553 model_params.model_info.team_public_model_name = original_model_name
1555 # Generate and assign unique internal model_name LAST
1556 # (after team_public_model_name is safely stored)
1557 unique_model_name: Final = f"model_name_{_team_id}_{uuid.uuid4()}"
1558 model_params.model_name = unique_model_name
1560 ## CREATE MODEL IN DB ##
1561 model_response: Final = await _add_model_to_db(
1562 model_params=model_params,
1563 user_api_key_dict=user_api_key_dict,
1564 prisma_client=prisma_client,
1565 slot=slot,
1566 )
1568 if original_model_name:
1569 await append_team_models(
1570 data=TeamModelAddRequest(
1571 team_id=_team_id,
1572 models=[original_model_name],
1573 ),
1574 prisma_client=prisma_client,
1575 user_api_key_cache=user_api_key_cache,
1576 proxy_logging_obj=proxy_logging_obj,
1577 )
1579 return model_response
1582async def _update_team_model_in_db(
1583 db_model: Deployment,
1584 patch_data: updateDeployment,
1585 user_api_key_dict: UserAPIKeyAuth,
1586 prisma_client: PrismaClient,
1587 write_row: Callable[[PrismaCompatibleUpdateDBModel], Awaitable[_RowT]],
1588) -> _RowT:
1589 """
1590 Handle team model updates with proper alias management.
1592 If patch_data contains a team_id:
1593 - Creates unique internal model_name and team alias
1594 - Adds model to team object
1595 - Preserves team_public_model_name for external reference
1597 The row is written through ``write_row`` before the team's model list is touched, so a
1598 refused or failed write leaves the team as it was (the create path orders itself the same way).
1599 """
1600 # Validate team_id if present in patch_data
1601 from litellm.proxy.proxy_server import premium_user
1603 await ModelManagementAuthChecks.allow_team_model_action(
1604 model_params=patch_data,
1605 user_api_key_dict=user_api_key_dict,
1606 prisma_client=prisma_client,
1607 premium_user=premium_user,
1608 )
1610 # Validated before the row write, beside the premium check the create path already runs here.
1611 #
1612 # The merged view is what gets stored, so that is what has to satisfy the invariants.
1613 # Validating the patch alone rejected a partial edit of an already valid deployment:
1614 # raising the rate on a configured model carries no ptu_effective_from, which the
1615 # stored row supplies.
1616 if patch_data.model_info is not None:
1617 _raise_if_ptu_cost_attribution_disabled(patch_data.model_info.model_dump(exclude_none=True))
1618 _validate_ptu_model_info(_merged_ptu_model_info(db_model=db_model, patch_data=patch_data))
1619 _raise_if_ptu_deployment_is_priced(
1620 model_info=_merged_ptu_model_info(db_model=db_model, patch_data=patch_data),
1621 supplied=(
1622 patch_data.litellm_params.model_dump(exclude_none=True) if patch_data.litellm_params else _EMPTY_MODEL_INFO
1623 ),
1624 )
1626 patch_team_id: Final = patch_data.model_info.team_id if patch_data.model_info else None
1628 # No team_id in patch, proceed with standard update
1629 if patch_team_id is None:
1630 return await write_row(update_db_model(db_model=db_model, updated_patch=patch_data))
1632 # Determine public model name
1633 public_model_name: Final = _get_public_model_name(
1634 patch_data=patch_data,
1635 db_model=db_model,
1636 )
1638 # Ensure model_info exists and set team_public_model_name
1639 if patch_data.model_info is None:
1640 from litellm.types.router import ModelInfo
1642 patch_data.model_info = ModelInfo()
1643 patch_data.model_info.team_public_model_name = public_model_name
1645 # Check if team assignment is new or changed
1646 db_team_id: Final = db_model.model_info.team_id if db_model.model_info else None
1647 is_new_team_assignment: Final = db_team_id != patch_team_id
1649 # Team rows keep their internal UUID-based model_name; the public name lives in model_info
1650 patch_data.model_name = f"model_name_{patch_team_id}_{uuid.uuid4()}" if is_new_team_assignment else None
1651 row: Final = await write_row(update_db_model(db_model=db_model, updated_patch=patch_data))
1653 if is_new_team_assignment:
1654 await _setup_new_team_model_assignment(
1655 team_id=patch_team_id,
1656 public_model_name=public_model_name,
1657 user_api_key_dict=user_api_key_dict,
1658 )
1659 else:
1660 await _update_existing_team_model_assignment(
1661 team_id=patch_team_id,
1662 public_model_name=public_model_name,
1663 db_model=db_model,
1664 user_api_key_dict=user_api_key_dict,
1665 prisma_client=prisma_client,
1666 )
1668 return row
1671def _get_public_model_name(
1672 patch_data: updateDeployment,
1673 db_model: Deployment,
1674) -> str:
1675 """Determine the public model name from patch or existing model.
1677 The top-level ``model_name`` is the rename channel. For team-scoped rows
1678 the DB ``model_name`` column holds an internal routing key
1679 (``model_name_{team_id}_{uuid}``), and ``/model/info`` historically leaked
1680 it into the dashboard edit form, so a non-rename save (e.g. a TPM tweak)
1681 would PATCH the internal name and the update path would treat it as a
1682 rename -- overwriting ``team_public_model_name`` and rewriting the team ACL
1683 (see issue #28382).
1685 Guard against that by ignoring an incoming ``model_name`` that matches the
1686 internal shape, or is a no-op against the current DB column. Anything else
1687 is a genuine rename and wins. We deliberately do NOT read
1688 ``patch_data.model_info.team_public_model_name``: the dashboard passes the
1689 existing ``model_info`` blob through untouched on a rename, so honoring it
1690 would return the OLD public name and silently drop the rename.
1692 Precedence (highest first):
1693 1. patch_data.model_name -- a genuine rename: not internal-shape and not a
1694 no-op against db_model.model_name.
1695 2. db_model.model_info.team_public_model_name -- existing public name.
1696 3. db_model.model_name -- last-resort fallback for legacy rows.
1697 """
1698 team_id: Final = (patch_data.model_info.team_id if patch_data.model_info else None) or (
1699 db_model.model_info.team_id if db_model.model_info else None
1700 )
1702 def _is_internal_shape(name: str | None) -> bool:
1703 if team_id is None or not name:
1704 return False
1705 return name.startswith(f"model_name_{team_id}_")
1707 incoming: Final = patch_data.model_name
1708 if incoming and not _is_internal_shape(incoming) and incoming != db_model.model_name:
1709 return incoming
1711 if db_model.model_info and db_model.model_info.team_public_model_name:
1712 return db_model.model_info.team_public_model_name
1714 return db_model.model_name
1717async def _setup_new_team_model_assignment(
1718 team_id: str,
1719 public_model_name: str,
1720 user_api_key_dict: UserAPIKeyAuth,
1721) -> None:
1722 """Register a newly team-assigned model's public name on the team."""
1723 await team_model_add(
1724 data=TeamModelAddRequest(
1725 team_id=team_id,
1726 models=[public_model_name],
1727 ),
1728 http_request=Request(scope={"type": "http"}),
1729 user_api_key_dict=user_api_key_dict,
1730 )
1733async def _get_team_deployments(
1734 team_id: str, prisma_client: PrismaClient, table: _ProxyModelTable | None = None
1735) -> Sequence[_ProxyModelRow]:
1736 """
1737 Fetch all deployments for a given team_id from the database.
1739 Centralizes team deployment queries to ensure consistent filtering and error handling.
1740 This is the established helper pattern for team deployment DB access in this module.
1742 Note: prisma-client-py 0.11.0 does not support JSON path filtering, so we filter
1743 by the model_name prefix (team models use "model_name_{team_id}_*") and confirm
1744 team_id in model_info with Python-side filtering.
1746 Pass ``table`` (a transaction's proxy-model table) to run the read inside an
1747 existing transaction.
1748 """
1749 prefix: Final = f"model_name_{team_id}_"
1750 table = table or _proxy_model_table(prisma_client)
1751 response: Final = await table.find_many(
1752 where={
1753 "model_name": {"startswith": prefix},
1754 }
1755 )
1756 if not response:
1757 return []
1759 # Confirm team_id in model_info (defensive check)
1760 result: Final = []
1761 for row in response:
1762 model_info = model_info_as_mapping(row.model_info)
1763 if model_info is not None and model_info.get("team_id") == team_id:
1764 result.append(row)
1765 return result
1768async def delete_team_models(
1769 team_ids: list[str],
1770 prisma_client: PrismaClient,
1771 llm_router: Router | None,
1772) -> list[str]:
1773 """
1774 Delete every BYOK model owned by the given teams, from the DB and the router.
1776 The DB rows are removed inside a single transaction, so deletion is atomic
1777 across all team_ids. Each team's rows are deleted by the exact model_ids read
1778 in the same transaction, which keeps the deleted set identical to the set
1779 handed to the router. The router is synced only after the transaction commits,
1780 so a rollback can never leave a deployment live in the router without its row.
1782 Returns the model_ids that were deleted.
1783 """
1784 deleted_model_ids: Final[list[str]] = []
1785 async with prisma_client.db.tx() as tx_ctx:
1786 tx: Final[_TxModelTables] = tx_ctx
1787 for team_id in team_ids: 1787 ↛ 1788line 1787 didn't jump to line 1788 because the loop on line 1787 never started
1788 rows = await _get_team_deployments(team_id, prisma_client, table=tx.litellm_proxymodeltable)
1789 model_ids = [row.model_id for row in rows]
1790 if model_ids:
1791 await tx.litellm_proxymodeltable.delete_many(where={"model_id": {"in": model_ids}})
1792 deleted_model_ids.extend(model_ids)
1794 if deleted_model_ids: 1794 ↛ 1795line 1794 didn't jump to line 1795 because the condition on line 1794 was never true
1795 await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable")
1797 # Under MODEL_RECONCILE_LOCK, for the same reason as delete_model: the rows are
1798 # gone, but a reconcile holding a pre-delete snapshot would upsert these ids back
1799 # onto this pod. The lock orders the eviction after any in-flight reconcile.
1800 from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK, proxy_config
1802 async with MODEL_RECONCILE_LOCK:
1803 proxy_config.remove_auto_router_catalog_entries(frozenset(deleted_model_ids))
1804 if llm_router is not None: 1804 ↛ 1808line 1804 didn't jump to line 1808
1805 for model_id in deleted_model_ids: 1805 ↛ 1806line 1805 didn't jump to line 1806 because the loop on line 1805 never started
1806 llm_router.delete_deployment(id=model_id)
1808 return deleted_model_ids
1811async def _get_team_public_model_names(
1812 team_id: str,
1813 prisma_client: PrismaClient,
1814) -> set[str]:
1815 """
1816 Public model names currently backed by a deployment in the team.
1818 Called on delete (after the deployment row is removed) so a public name that is
1819 load-balanced across several deployments stays in team.models while a replica
1820 still serves it.
1821 """
1822 deployments: Final = await _get_team_deployments(team_id, prisma_client)
1823 public_names: Final[set[str]] = set()
1824 for row in deployments:
1825 model_info = model_info_as_mapping(row.model_info)
1826 if model_info is not None:
1827 public_name = model_info.get("team_public_model_name")
1828 if public_name:
1829 public_names.add(public_name)
1830 return public_names
1833async def _remove_unbacked_team_models(
1834 model_params: Deployment,
1835 prisma_client: PrismaClient,
1836 user_api_key_cache: UserApiKeyCache,
1837 proxy_logging_obj: ProxyLogging,
1838 llm_router: Router | None = None,
1839) -> None:
1840 """
1841 Strip a deleted team model's public name(s) from team.models and refresh the cache.
1843 Must be called after the deployment row is deleted: a public name is removed only
1844 when no remaining team deployment still backs it, so a load-balanced replica isn't
1845 revoked while siblings serve it, and concurrent deletes can't leave a ghost.
1847 Legacy team models (created before team_public_model_name existed) store a
1848 ``{public_name: "model_name_{team_id}_{uuid}"}`` entry in the team's model_aliases,
1849 so the alias scan runs for every team model; skipping it for internal-shaped names
1850 left stale aliases that rewrote requests to deployments that no longer exist.
1851 Aliases are scrubbed only when the deleted deployment's name no longer resolves in
1852 the router, so deleting one replica of a load-balanced group never breaks aliases
1853 that still route to the surviving replicas (in any team).
1855 A public name that still resolves to a live router deployment (e.g. a gateway-level
1856 model group shared with the team) is kept in team.models, so deleting a per-team
1857 duplicate does not revoke the team's access to the shared deployment.
1858 """
1859 team_id: Final = model_params.model_info.team_id
1860 if team_id is None:
1861 return
1863 deleted_name_still_served: Final = (
1864 llm_router is not None and model_params.model_name in llm_router.model_name_to_deployment_indices
1865 )
1866 removed_model_aliases: Final[list[tuple[str, str]]] = (
1867 []
1868 if deleted_name_still_served
1869 else await delete_team_model_alias(
1870 public_model_name=model_params.model_name,
1871 prisma_client=prisma_client,
1872 )
1873 )
1874 removed_alias_names: Final = {alias for alias_team_id, alias in removed_model_aliases if alias_team_id == team_id}
1875 candidate_names: Final = (
1876 removed_alias_names | {model_params.model_info.team_public_model_name}
1877 if model_params.model_info.team_public_model_name is not None
1878 else removed_alias_names
1879 )
1880 if not candidate_names:
1881 return
1883 team_backed_names: Final = await _get_team_public_model_names(team_id=team_id, prisma_client=prisma_client)
1884 router_served_names: Final = (
1885 frozenset(name for name in candidate_names if name in llm_router.model_name_to_deployment_indices)
1886 if llm_router is not None
1887 else frozenset()
1888 )
1889 names_to_remove: Final = candidate_names - team_backed_names - router_served_names
1890 if not names_to_remove:
1891 return
1893 existing_team_row: Final = await _db_team_table(prisma_client).find_unique(where={"team_id": team_id})
1894 if existing_team_row is None:
1895 return
1897 updated_team_row: Final[LiteLLM_TeamTable] = await _db_team_table(prisma_client).update(
1898 where={"team_id": team_id},
1899 data={"models": [model for model in existing_team_row.models if model not in names_to_remove]},
1900 include={"object_permission": True},
1901 )
1902 await _refresh_cached_team(
1903 team_row=updated_team_row,
1904 user_api_key_cache=user_api_key_cache,
1905 proxy_logging_obj=proxy_logging_obj,
1906 )
1909async def _update_existing_team_model_assignment(
1910 team_id: str,
1911 public_model_name: str,
1912 db_model: Deployment,
1913 user_api_key_dict: UserAPIKeyAuth,
1914 prisma_client: PrismaClient | None,
1915) -> None:
1916 """Update an existing team model if the public name changed.
1918 Note on DB scan: Prisma's JSON filtering does not support compound AND conditions
1919 across multiple JSON paths, so we fetch all deployments for the team and filter
1920 team_public_model_name in Python. For teams with many deployments this scan grows
1921 linearly; if team deployment counts become large this should be revisited.
1922 """
1924 def _get_team_public_model_name(
1925 model_info: object,
1926 ) -> str | None:
1927 parsed: Final = model_info_as_mapping(model_info)
1928 if parsed is None:
1929 return None
1930 value: Final = parsed.get("team_public_model_name")
1931 return value if isinstance(value, str) else None
1933 old_public_name: Final = db_model.model_info.team_public_model_name if db_model.model_info else None
1935 if old_public_name and public_model_name != old_public_name:
1936 if prisma_client is None:
1937 verbose_proxy_logger.warning(
1938 "prisma_client not initialized; skipping public name update entirely to avoid orphaned entries"
1939 )
1940 return
1942 # Query DB for all team deployments to check for sibling deployments
1943 team_deployments: Final = await _get_team_deployments(team_id, prisma_client)
1944 other_deployments_with_old_name: Final = [
1945 d
1946 for d in team_deployments
1947 if d.model_name != db_model.model_name and _get_team_public_model_name(d.model_info) == old_public_name
1948 ]
1950 # Add new name first, then delete old name to prevent access loss on partial failure
1951 await team_model_add(
1952 data=TeamModelAddRequest(
1953 team_id=team_id,
1954 models=[public_model_name],
1955 ),
1956 http_request=Request(scope={"type": "http"}),
1957 user_api_key_dict=user_api_key_dict,
1958 )
1960 if not other_deployments_with_old_name:
1961 await team_model_delete(
1962 data=TeamModelDeleteRequest(
1963 team_id=team_id,
1964 models=[old_public_name],
1965 ),
1966 http_request=Request(scope={"type": "http"}),
1967 user_api_key_dict=user_api_key_dict,
1968 )
1969 elif not old_public_name and public_model_name:
1970 # First-time assignment of public name on an existing team deployment:
1971 # ensure the team's models list is updated so team routing can resolve it.
1972 await team_model_add(
1973 data=TeamModelAddRequest(
1974 team_id=team_id,
1975 models=[public_model_name],
1976 ),
1977 http_request=Request(scope={"type": "http"}),
1978 user_api_key_dict=user_api_key_dict,
1979 )
1980 # else: old_public_name == public_model_name (no rename needed)
1981 # No team_model_add/delete calls required; public name is already registered
1984def _canonical_session_tags(tags: Sequence[AwsSessionTag]) -> tuple[tuple[str, str], ...]:
1985 return tuple(sorted((tag["Key"], tag["Value"]) for tag in tags))
1988class ModelManagementAuthChecks:
1989 """
1990 Common auth checks for model management endpoints
1991 """
1993 @staticmethod
1994 def can_user_make_team_model_call(
1995 team_id: str,
1996 user_api_key_dict: UserAPIKeyAuth,
1997 team_obj: LiteLLM_TeamTable | None = None,
1998 premium_user: bool = False,
1999 ) -> Literal[True]:
2000 if premium_user is False:
2001 raise HTTPException(
2002 status_code=403,
2003 detail={"error": CommonProxyErrors.not_premium_user.value},
2004 )
2005 if user_api_key_dict.user_role and user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
2006 return True
2007 elif team_obj is None or not _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team_obj):
2008 raise HTTPException(
2009 status_code=403,
2010 detail={
2011 "error": f"Team ID={team_id} does not match the API key's team ID={user_api_key_dict.team_id}, OR you are not the admin for this team. Check `/user/info` to verify your team admin status."
2012 },
2013 )
2014 return True
2016 @staticmethod
2017 def can_user_attach_credential(
2018 litellm_params: GenericLiteLLMParams | None,
2019 user_api_key_dict: UserAPIKeyAuth,
2020 existing_litellm_params: GenericLiteLLMParams | None = None,
2021 *,
2022 null_detaches: bool = False,
2023 ) -> Literal[True]:
2024 if litellm_params is None: 2024 ↛ 2025line 2024 didn't jump to line 2025 because the condition on line 2024 was never true
2025 return True
2026 if "litellm_credential_name" not in litellm_params.model_fields_set:
2027 return True
2028 if litellm_params.litellm_credential_name is None and not null_detaches: 2028 ↛ 2030line 2028 didn't jump to line 2030 because the condition on line 2028 was always true
2029 return True
2030 requested_credential_name: Final = litellm_params.litellm_credential_name
2031 if requested_credential_name == _stored_credential_name(existing_litellm_params):
2032 return True
2033 if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
2034 return True
2035 action: Final = "detach" if requested_credential_name is None else "attach"
2036 raise ProxyException(
2037 message=f"Only a proxy admin can {action} a stored credential (litellm_credential_name) on a model. Your role={user_api_key_dict.user_role}.",
2038 type=ProxyErrorTypes.auth_error.value,
2039 code=status.HTTP_403_FORBIDDEN,
2040 param="litellm_credential_name",
2041 )
2043 @staticmethod
2044 def can_user_set_aws_session_tags(
2045 litellm_params: GenericLiteLLMParams | None,
2046 user_api_key_dict: UserAPIKeyAuth,
2047 existing_litellm_params: GenericLiteLLMParams | None = None,
2048 ) -> Literal[True]:
2049 if litellm_params is None or litellm_params.aws_session_tags is None:
2050 return True
2051 if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN: 2051 ↛ 2053line 2051 didn't jump to line 2053 because the condition on line 2051 was always true
2052 return True
2053 existing_tags: Final = existing_litellm_params.aws_session_tags if existing_litellm_params is not None else None
2054 if existing_tags is not None and _canonical_session_tags(existing_tags) == _canonical_session_tags(
2055 litellm_params.aws_session_tags
2056 ):
2057 return True
2058 raise ProxyException(
2059 message=f"Only a proxy admin can set aws_session_tags on a model. Your role={user_api_key_dict.user_role}.",
2060 type=ProxyErrorTypes.auth_error.value,
2061 code=status.HTTP_403_FORBIDDEN,
2062 param="aws_session_tags",
2063 )
2065 @staticmethod
2066 async def allow_team_model_action(
2067 model_params: Deployment | updateDeployment,
2068 user_api_key_dict: UserAPIKeyAuth,
2069 prisma_client: PrismaClient,
2070 premium_user: bool,
2071 ) -> Literal[True]:
2072 if model_params.model_info is None or model_params.model_info.team_id is None:
2073 return True
2074 if model_params.model_info.team_id is not None and premium_user is not True:
2075 raise HTTPException(
2076 status_code=403,
2077 detail={"error": CommonProxyErrors.not_premium_user.value},
2078 )
2080 _existing_team_row: Final = await _repo_team_table(prisma_client).find_unique(
2081 where={"team_id": model_params.model_info.team_id}
2082 )
2084 if _existing_team_row is None:
2085 raise HTTPException(
2086 status_code=400,
2087 detail={"error": f"Team id={model_params.model_info.team_id} does not exist in db"},
2088 )
2089 existing_team_row: Final = LiteLLM_TeamTable.model_validate(_existing_team_row.model_dump())
2091 ModelManagementAuthChecks.can_user_make_team_model_call(
2092 team_id=model_params.model_info.team_id,
2093 user_api_key_dict=user_api_key_dict,
2094 team_obj=existing_team_row,
2095 premium_user=premium_user,
2096 )
2097 return True
2099 @staticmethod
2100 async def can_user_make_model_call(
2101 model_params: Deployment,
2102 user_api_key_dict: UserAPIKeyAuth,
2103 prisma_client: PrismaClient,
2104 premium_user: bool,
2105 allow_missing_team: bool = False,
2106 member_operation: Literal["create", "update"] | None = None,
2107 incoming_model_params: updateDeployment | None = None,
2108 ) -> Literal[True] | MemberAutoRouterWrite:
2109 if user_api_key_dict.user_role in ( 2109 ↛ 2113line 2109 didn't jump to line 2113 because the condition on line 2109 was never true
2110 LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
2111 LitellmUserRoles.INTERNAL_USER_VIEW_ONLY,
2112 ):
2113 raise HTTPException(status_code=403, detail="View-only users cannot manage models.")
2114 ## Check team model auth
2115 if model_params.model_info.team_id is not None: 2115 ↛ 2116line 2115 didn't jump to line 2116 because the condition on line 2115 was never true
2116 team_obj_row: Final = await _repo_team_table(prisma_client).find_unique(
2117 where={"team_id": model_params.model_info.team_id}
2118 )
2119 if team_obj_row is None:
2120 # The team was deleted. Callers that opt in (e.g. model deletion) may
2121 # act on the orphaned model, but only as a proxy admin -- without the
2122 # team there is no team-admin membership left to verify.
2123 if allow_missing_team:
2124 if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
2125 return True
2126 raise HTTPException(
2127 status_code=403,
2128 detail={"error": "Only a proxy admin can delete a model whose team has been deleted."},
2129 )
2130 raise HTTPException(
2131 status_code=400,
2132 detail={"error": f"Team id={model_params.model_info.team_id} does not exist in db"},
2133 )
2134 team_obj: Final = LiteLLM_TeamTable.model_validate(team_obj_row.model_dump())
2136 if (
2137 member_operation is not None
2138 and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN
2139 and not _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team_obj)
2140 ):
2141 from litellm.proxy.proxy_server import llm_router
2143 if llm_router is None or (member_operation == "update" and incoming_model_params is None):
2144 raise HTTPException(
2145 status_code=400, detail="An auto-router configuration and model catalog are required."
2146 )
2147 return await authorize_member_auto_router_write(
2148 incoming=incoming_model_params if incoming_model_params is not None else model_params,
2149 existing=model_params if member_operation == "update" else None,
2150 user_api_key_dict=user_api_key_dict,
2151 team=team_obj,
2152 premium_user=premium_user,
2153 prisma_client=prisma_client,
2154 llm_router=llm_router,
2155 )
2157 return ModelManagementAuthChecks.can_user_make_team_model_call(
2158 team_id=model_params.model_info.team_id,
2159 user_api_key_dict=user_api_key_dict,
2160 team_obj=team_obj,
2161 premium_user=premium_user,
2162 )
2163 ## Check non-team model auth
2164 elif user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: 2164 ↛ 2165line 2164 didn't jump to line 2165 because the condition on line 2164 was never true
2165 raise HTTPException(
2166 status_code=403,
2167 detail={
2168 "error": f"User does not have permission to make this model call. Your role={user_api_key_dict.user_role}. You can only make model calls if you are a PROXY_ADMIN or if you are a team admin, by specifying a team_id in the model_info."
2169 },
2170 )
2171 else:
2172 return True
2174 return True
2177#### [BETA] - This is a beta endpoint, format might change based on user feedback. - https://github.com/BerriAI/litellm/issues/964
2178@router.post(
2179 "/model/delete",
2180 description="Allows deleting models in the model list in the config.yaml",
2181 tags=["model management"],
2182 dependencies=[Depends(user_api_key_auth)],
2183)
2184async def delete_model(
2185 model_info: ModelInfoDelete,
2186 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
2187):
2188 from litellm.proxy.proxy_server import llm_router
2190 try:
2191 """
2192 [BETA] - This is a beta endpoint, format might change based on user feedback. - https://github.com/BerriAI/litellm/issues/964
2194 - Check if id in db
2195 - Delete
2196 """
2198 from litellm.proxy.proxy_server import (
2199 MODEL_RECONCILE_LOCK,
2200 llm_router,
2201 premium_user,
2202 prisma_client,
2203 proxy_config,
2204 proxy_logging_obj,
2205 store_model_in_db,
2206 user_api_key_cache,
2207 )
2209 if prisma_client is None: 2209 ↛ 2210line 2209 didn't jump to line 2210 because the condition on line 2209 was never true
2210 raise HTTPException(
2211 status_code=500,
2212 detail={
2213 "error": "No DB Connected. Here's how to do it - https://docs.litellm.ai/docs/proxy/virtual_keys"
2214 },
2215 )
2217 model_in_db: Final = await _proxy_model_table(prisma_client).find_unique(where={"model_id": model_info.id})
2218 if model_in_db is None:
2219 raise HTTPException(
2220 status_code=400,
2221 detail={"error": f"Model with id={model_info.id} not found in db"},
2222 )
2224 model_params: Final = Deployment(**model_in_db.model_dump())
2225 await ModelManagementAuthChecks.can_user_make_model_call(
2226 model_params=model_params,
2227 user_api_key_dict=user_api_key_dict,
2228 prisma_client=prisma_client,
2229 premium_user=premium_user,
2230 allow_missing_team=True,
2231 )
2233 # update DB
2234 if store_model_in_db is True: 2234 ↛ 2291line 2234 didn't jump to line 2291 because the condition on line 2234 was always true
2235 """
2236 - store model_list in db
2237 - store keys separately
2238 """
2239 # encrypt litellm params #
2240 result: Final = await _proxy_model_table(prisma_client).delete(where={"model_id": model_info.id})
2242 if result is None: 2242 ↛ 2243line 2242 didn't jump to line 2243 because the condition on line 2242 was never true
2243 raise HTTPException(
2244 status_code=400,
2245 detail={"error": f"Model with id={model_info.id} not found in db"},
2246 )
2248 ## DELETE FROM ROUTER ##
2249 # Under MODEL_RECONCILE_LOCK. The db row is already gone, but a reconcile
2250 # that snapshotted the db BEFORE that delete still lists this id as desired,
2251 # and its _add_deployment upserts the deployment straight back -- leaving
2252 # this pod serving a model the database no longer has, until the next
2253 # reconcile. Taking the lock orders this eviction after any such in-flight
2254 # reconcile's re-add, so the eviction is the last word.
2255 async with MODEL_RECONCILE_LOCK:
2256 proxy_config.remove_auto_router_catalog_entries(frozenset({model_info.id}))
2257 if llm_router is not None: 2257 ↛ 2261line 2257 didn't jump to line 2261
2258 llm_router.delete_deployment(id=model_info.id)
2260 # Runs after the row delete so the sibling check sees post-delete state.
2261 if model_params.model_info.team_id is not None: 2261 ↛ 2262line 2261 didn't jump to line 2262 because the condition on line 2261 was never true
2262 await _remove_unbacked_team_models(
2263 model_params=model_params,
2264 prisma_client=prisma_client,
2265 user_api_key_cache=user_api_key_cache,
2266 proxy_logging_obj=proxy_logging_obj,
2267 llm_router=llm_router,
2268 )
2269 await sync_access_groups_for_deleted_model(
2270 prisma_client=prisma_client,
2271 model_id=model_info.id,
2272 model_name=model_params.model_name,
2273 llm_router=llm_router,
2274 )
2276 ## CREATE AUDIT LOG ##
2277 asyncio.create_task(
2278 create_object_audit_log(
2279 object_id=model_info.id,
2280 action="deleted",
2281 user_api_key_dict=user_api_key_dict,
2282 table_name=LitellmTableNames.PROXY_MODEL_TABLE_NAME,
2283 before_value=result.model_dump_json(exclude_none=True),
2284 after_value=None,
2285 litellm_changed_by=user_api_key_dict.user_id,
2286 litellm_proxy_admin_name=LITELLM_PROXY_ADMIN_NAME,
2287 )
2288 )
2289 return {"message": f"Model: {result.model_id} deleted successfully"}
2290 else:
2291 raise HTTPException(
2292 status_code=500,
2293 detail={"error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."},
2294 )
2296 except Exception as e:
2297 verbose_proxy_logger.exception("Failed to delete model. Due to error - %s", e)
2298 if isinstance(e, HTTPException): 2298 ↛ 2305line 2298 didn't jump to line 2305 because the condition on line 2298 was always true
2299 raise ProxyException(
2300 message=getattr(e, "detail", f"Authentication Error({e})"),
2301 type=ProxyErrorTypes.auth_error,
2302 param=getattr(e, "param", "None"),
2303 code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
2304 )
2305 elif isinstance(e, ProxyException):
2306 raise e
2307 raise ProxyException(
2308 message="Authentication Error, " + str(e),
2309 type=ProxyErrorTypes.auth_error,
2310 param=getattr(e, "param", "None"),
2311 code=status.HTTP_400_BAD_REQUEST,
2312 )
2315async def delete_team_model_alias(
2316 public_model_name: str,
2317 prisma_client: PrismaClient,
2318) -> list[tuple[str, str]]:
2319 """
2320 Delete a team model alias
2322 Iterate through all team model aliases and delete the one that matches the model_id
2324 Returns:
2325 - List of team id + model alias pairs that were removed
2326 """
2327 team_model_aliases: Final = await _model_alias_table(prisma_client).find_many(include={"team": True})
2328 tasks: Final = []
2329 removed_model_aliases: Final[list[tuple[str, str]]] = []
2330 for team_model_alias in team_model_aliases:
2331 model_aliases = cast( # cast-ok: prisma types Json columns as `str`; the driver hands back the parsed dict
2332 "dict[str, str]", team_model_alias.model_aliases
2333 )
2334 id = team_model_alias.id
2336 if public_model_name in model_aliases.values():
2337 key = list(model_aliases.keys())[list(model_aliases.values()).index(public_model_name)]
2338 if team_model_alias.team is not None:
2339 removed_model_aliases.append((team_model_alias.team.team_id, key))
2340 del model_aliases[key]
2341 tasks.append(
2342 _model_alias_table(prisma_client).update(
2343 where={"id": id},
2344 data={"model_aliases": json.dumps(model_aliases)},
2345 )
2346 )
2347 await asyncio.gather(*tasks)
2349 return removed_model_aliases
2352#### [BETA] - This is a beta endpoint, format might change based on user feedback. - https://github.com/BerriAI/litellm/issues/964
2353@router.post(
2354 "/model/new",
2355 description="Allows adding new models to the model list in the config.yaml",
2356 tags=["model management"],
2357 dependencies=[Depends(user_api_key_auth)],
2358)
2359async def add_new_model(
2360 model_params: Deployment,
2361 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
2362):
2363 """
2364 Add a new model to the proxy.
2366 Parameters:
2367 - model_name: str - The name users will use to call this model (required)
2368 - litellm_params: dict - LiteLLM-specific parameters (required)
2369 - model: str - The actual model identifier, e.g., "azure/my-deployment-name" (required - this is the only required field in litellm_params)
2370 - api_key: str - API key for the provider (optional)
2371 - api_base: str - API base URL (optional)
2372 - Other optional params: api_version, timeout, max_retries, etc.
2373 - model_info: dict - Additional model metadata returned in /v1/model/info (optional)
2375 Example curl:
2377 ```bash
2378 curl -L -X POST 'http://0.0.0.0:4000/model/new' \
2379 -H 'Authorization: Bearer LITELLM_VIRTUAL_KEY' \
2380 -H 'Content-Type: application/json' \
2381 -d '{
2382 "model_name": "my-azure-model",
2383 "litellm_params": {
2384 "model": "azure/my-deployment-name",
2385 "api_key": "my-azure-api-key",
2386 "api_base": "https://my-endpoint.openai.azure.com"
2387 },
2388 "model_info": {
2389 "my_custom_key": "my_custom_value"
2390 }
2391 }'
2392 ```
2394 Returns:
2395 - The created model entry with model_id
2396 """
2397 from litellm.proxy.proxy_server import (
2398 general_settings,
2399 premium_user,
2400 prisma_client,
2401 proxy_config,
2402 proxy_logging_obj,
2403 store_model_in_db,
2404 )
2406 try:
2407 if prisma_client is None: 2407 ↛ 2408line 2407 didn't jump to line 2408 because the condition on line 2407 was never true
2408 raise HTTPException(
2409 status_code=500,
2410 detail={
2411 "error": "No DB Connected. Here's how to do it - https://docs.litellm.ai/docs/proxy/virtual_keys"
2412 },
2413 )
2415 ## Auth check
2416 write_authorization: Final = await ModelManagementAuthChecks.can_user_make_model_call(
2417 model_params=model_params,
2418 user_api_key_dict=user_api_key_dict,
2419 prisma_client=prisma_client,
2420 premium_user=premium_user,
2421 member_operation="create",
2422 )
2423 member_write: Final = write_authorization if isinstance(write_authorization, MemberAutoRouterWrite) else None
2425 ModelManagementAuthChecks.can_user_attach_credential(
2426 litellm_params=model_params.litellm_params,
2427 user_api_key_dict=user_api_key_dict,
2428 )
2430 ModelManagementAuthChecks.can_user_set_aws_session_tags(
2431 litellm_params=model_params.litellm_params,
2432 user_api_key_dict=user_api_key_dict,
2433 )
2435 _raise_on_strategy_router_write_violation(
2436 incoming_params=model_params.litellm_params,
2437 existing_params=None,
2438 )
2440 _raise_if_rate_limits_required_but_missing(
2441 litellm_params=model_params.litellm_params,
2442 enforced=bool(general_settings.get(ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING, False)),
2443 )
2445 clean_model_info: Final = ModelInfo(
2446 **without_server_derived_pricing(model_params.model_info.model_dump(exclude_none=True))
2447 )
2448 model_params.model_info = ( # rebind-ok: downstream team-model handling mutates this same object
2449 clean_model_info.model_copy(update=MappingProxyType({"member_auto_router": True}))
2450 if member_write is not None
2451 else clean_model_info
2452 )
2454 model_response: prisma_models.LiteLLM_ProxyModelTable | LiteLLM_ProxyModelTable | None = None
2455 # update DB
2456 incoming_model_info: Final = model_params.model_info.model_dump(exclude_none=True)
2457 _raise_if_ptu_cost_attribution_disabled(incoming_model_info)
2458 _validate_ptu_model_info(incoming_model_info)
2459 priced_model_params: Final = _ptu_priced_deployment(model_params)
2461 if store_model_in_db is True: 2461 ↛ 2506line 2461 didn't jump to line 2506 because the condition on line 2461 was always true
2462 """
2463 - store model_list in db
2464 - store keys separately
2465 """
2467 live_before_reload: Final = live_model_ids_snapshot()
2468 reload_outcome: ReconcileOutcome = ReconcileOutcome(still_desired=None, live_after=None)
2469 try:
2470 _original_litellm_model_name: Final = model_params.model_name
2471 add_model: Final = (
2472 _add_model_to_db if model_params.model_info.team_id is None else _add_team_model_to_db
2473 )
2474 model_response = await add_model(
2475 model_params=priced_model_params,
2476 user_api_key_dict=user_api_key_dict,
2477 prisma_client=prisma_client,
2478 slot=_auto_router_capability_slot(
2479 prisma_client,
2480 effective_params=_effective_complexity_router_params(
2481 priced_model_params.litellm_params,
2482 None,
2483 ),
2484 model_id=priced_model_params.model_info.id,
2485 member_write=member_write,
2486 ),
2487 )
2488 reload_outcome = await proxy_config.add_deployment(
2489 prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
2490 )
2491 # don't let failed slack alert block the /model/new response
2492 _alerting: Final = general_settings.get("alerting", []) or []
2493 if "slack" in _alerting: 2493 ↛ 2495line 2493 didn't jump to line 2495 because the condition on line 2493 was never true
2494 # send notification - new model added
2495 await proxy_logging_obj.slack_alerting_instance.model_added_alert(
2496 model_name=priced_model_params.model_name,
2497 litellm_model_name=_original_litellm_model_name,
2498 passed_model_info=priced_model_params.model_info,
2499 )
2500 except Exception as e:
2501 if isinstance(e, HTTPException): 2501 ↛ 2502line 2501 didn't jump to line 2502 because the condition on line 2501 was never true
2502 raise
2503 verbose_proxy_logger.exception("Exception in add_new_model: %s", e)
2505 else:
2506 raise HTTPException(
2507 status_code=500,
2508 detail={"error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."},
2509 )
2511 if model_response is None:
2512 raise HTTPException(
2513 status_code=500,
2514 detail={"error": "Failed to add model to db. Check your server logs for more details."},
2515 )
2517 ## CREATE AUDIT LOG ##
2518 asyncio.create_task(
2519 create_object_audit_log(
2520 object_id=model_response.model_id,
2521 action="created",
2522 user_api_key_dict=user_api_key_dict,
2523 table_name=LitellmTableNames.PROXY_MODEL_TABLE_NAME,
2524 before_value=None,
2525 after_value=(
2526 model_response.model_dump_json(exclude_none=True) if isinstance(model_response, BaseModel) else None
2527 ),
2528 litellm_changed_by=user_api_key_dict.user_id,
2529 litellm_proxy_admin_name=LITELLM_PROXY_ADMIN_NAME,
2530 )
2531 )
2533 raise_if_reload_degraded_serving(
2534 before=live_before_reload,
2535 written_models=[(model_response.model_id, getattr(model_response, "model_info", None))],
2536 action="create",
2537 still_desired=reload_outcome.still_desired,
2538 live_after=reload_outcome.live_after,
2539 )
2541 return model_response
2543 except Exception as e:
2544 verbose_proxy_logger.exception("litellm.proxy.proxy_server.add_new_model(): Exception occured - %s", e)
2545 if isinstance(e, HTTPException):
2546 raise ProxyException(
2547 message=getattr(e, "detail", f"Authentication Error({e})"),
2548 type=ProxyErrorTypes.auth_error,
2549 param=getattr(e, "param", "None"),
2550 code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
2551 )
2552 elif isinstance(e, ProxyException): 2552 ↛ 2554line 2552 didn't jump to line 2554 because the condition on line 2552 was always true
2553 raise e
2554 raise ProxyException(
2555 message="Authentication Error, " + str(e),
2556 type=ProxyErrorTypes.auth_error,
2557 param=getattr(e, "param", "None"),
2558 code=status.HTTP_400_BAD_REQUEST,
2559 )
2562#### MODEL MANAGEMENT ####
2563@router.post(
2564 "/model/update",
2565 description="Edit existing model params",
2566 tags=["model management"],
2567 dependencies=[Depends(user_api_key_auth)],
2568)
2569async def update_model(
2570 model_params: updateDeployment,
2571 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
2572):
2573 """
2574 Old endpoint for model update. Makes a PUT request.
2576 Use `/model/{model_id}/update` to PATCH the stored model in db.
2577 """
2578 from litellm.proxy.proxy_server import (
2579 LITELLM_PROXY_ADMIN_NAME,
2580 llm_router,
2581 premium_user,
2582 prisma_client,
2583 store_model_in_db,
2584 user_api_key_cache,
2585 )
2587 try:
2588 if prisma_client is None: 2588 ↛ 2589line 2588 didn't jump to line 2589 because the condition on line 2588 was never true
2589 raise HTTPException(
2590 status_code=500,
2591 detail={
2592 "error": "No DB Connected. Here's how to do it - https://docs.litellm.ai/docs/proxy/virtual_keys"
2593 },
2594 )
2596 _model_id: str | None = None
2597 _model_info: Final[ModelInfo | None] = getattr(model_params, "model_info", None)
2598 if _model_info is None:
2599 raise Exception("model_info not provided")
2601 _model_id = _model_info.id
2602 if _model_id is None: 2602 ↛ 2603line 2602 didn't jump to line 2603 because the condition on line 2602 was never true
2603 raise Exception("model_info.id not provided")
2605 _existing_litellm_params = await ModelRepository(prisma_client).table.find_unique(where={"model_id": _model_id})
2607 if _existing_litellm_params is None:
2608 if llm_router is not None and llm_router.get_deployment(model_id=_model_id) is not None: 2608 ↛ 2609line 2608 didn't jump to line 2609 because the condition on line 2608 was never true
2609 raise HTTPException(
2610 status_code=400,
2611 detail={"error": "Can't edit model. Model in config. Store model in db via `/model/new`. to edit."},
2612 )
2613 else:
2614 raise Exception("model not found")
2615 deployment: Final = Deployment(**_existing_litellm_params.model_dump())
2617 write_authorization: Final = await ModelManagementAuthChecks.can_user_make_model_call(
2618 model_params=deployment,
2619 user_api_key_dict=user_api_key_dict,
2620 prisma_client=prisma_client,
2621 premium_user=premium_user,
2622 member_operation="update",
2623 incoming_model_params=model_params,
2624 )
2625 member_write: Final = write_authorization if isinstance(write_authorization, MemberAutoRouterWrite) else None
2627 ModelManagementAuthChecks.can_user_attach_credential(
2628 litellm_params=model_params.litellm_params,
2629 user_api_key_dict=user_api_key_dict,
2630 existing_litellm_params=deployment.litellm_params,
2631 )
2633 ModelManagementAuthChecks.can_user_set_aws_session_tags(
2634 litellm_params=model_params.litellm_params,
2635 user_api_key_dict=user_api_key_dict,
2636 existing_litellm_params=deployment.litellm_params,
2637 )
2639 _raise_on_strategy_router_write_violation(
2640 incoming_params=model_params.litellm_params,
2641 existing_params=deployment.litellm_params,
2642 )
2643 effective_params: Final = _effective_complexity_router_params(
2644 model_params.litellm_params, deployment.litellm_params
2645 )
2646 member_marker: Final = _member_auto_router_marker_for_update(
2647 incoming_params=model_params.litellm_params, existing=deployment, member_write=member_write
2648 )
2650 # update DB
2651 if store_model_in_db is True: 2651 ↛ exitline 2651 didn't return from function 'update_model' because the condition on line 2651 was always true
2652 existing_model_row: Final = cast( # cast-ok: prisma types Json columns as `str`; the driver parses them
2653 "_ExistingModelRow", _existing_litellm_params
2654 )
2655 _existing_litellm_params_dict: Final = dict(existing_model_row.litellm_params)
2657 if model_params.litellm_params is None: 2657 ↛ 2658line 2657 didn't jump to line 2658 because the condition on line 2657 was never true
2658 raise Exception("litellm_params not provided")
2660 _new_litellm_params_dict: Final = model_params.litellm_params.dict(exclude_none=True)
2662 ### ENCRYPT PARAMS ###
2663 encrypted_params: Final = MappingProxyType(
2664 {
2665 k: (
2666 _effective_complexity_router_config(model_params.litellm_params, deployment.litellm_params)
2667 if k == "complexity_router_config"
2668 else encrypt_value_helper(value=v)
2669 )
2670 for k, v in _new_litellm_params_dict.items()
2671 }
2672 )
2674 ### MERGE WITH EXISTING DATA ###
2675 _mp: Final[dict[str, object]] = model_params.litellm_params.dict()
2676 merged_dictionary: Final = {
2677 key: _existing_litellm_params_dict[key] if value is None else encrypted_params[key]
2678 for key, value in _mp.items()
2679 if value is not None or _existing_litellm_params_dict.get(key) is not None
2680 }
2682 renamed_to: Final = (
2683 model_params.model_name
2684 if model_params.model_name not in (None, deployment.model_name)
2685 and deployment.model_info.team_id is None
2686 else None
2687 )
2688 base_update: Final[PrismaCompatibleUpdateDBModel] = {
2689 "litellm_params": json.dumps(merged_dictionary),
2690 "updated_by": user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME,
2691 }
2692 renamed_update: Final[PrismaCompatibleUpdateDBModel] = (
2693 {**base_update, "model_name": renamed_to} # mutable-ok: Prisma serializes only concrete update dicts
2694 if renamed_to is not None
2695 else base_update
2696 )
2697 _data: Final[PrismaCompatibleUpdateDBModel] = (
2698 { # mutable-ok: Prisma serializes only concrete update dicts
2699 **renamed_update,
2700 "model_info": deployment.model_info.model_copy(
2701 update=MappingProxyType({"member_auto_router": member_marker})
2702 ).model_dump_json(exclude_none=True),
2703 }
2704 if member_marker is not None
2705 else renamed_update
2706 )
2707 async with _auto_router_capability_slot(
2708 prisma_client,
2709 effective_params=effective_params,
2710 model_id=_model_id,
2711 member_write=member_write,
2712 ) as table:
2713 model_response: Final = await table.update(
2714 where={"model_id": _model_id},
2715 data=_data,
2716 )
2717 if renamed_to is not None:
2718 await sync_access_groups_for_renamed_model(
2719 prisma_client=prisma_client,
2720 model_id=_model_id,
2721 old_name=deployment.model_name,
2722 new_name=renamed_to,
2723 llm_router=llm_router,
2724 )
2725 await sync_model_allowlists_for_renamed_model(
2726 prisma_client=prisma_client,
2727 model_id=_model_id,
2728 old_name=deployment.model_name,
2729 new_name=renamed_to,
2730 llm_router=llm_router,
2731 user_api_key_cache=user_api_key_cache,
2732 )
2734 # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates)
2735 live_before_reload: Final = live_model_ids_snapshot()
2736 reload_outcome: Final = await clear_cache()
2737 ## CREATE AUDIT LOG ##
2738 asyncio.create_task(
2739 create_object_audit_log(
2740 object_id=_model_id,
2741 action="updated",
2742 user_api_key_dict=user_api_key_dict,
2743 table_name=LitellmTableNames.PROXY_MODEL_TABLE_NAME,
2744 before_value=(
2745 existing_model_row.model_dump_json(exclude_none=True)
2746 if isinstance(existing_model_row, BaseModel)
2747 else None
2748 ),
2749 after_value=(
2750 model_response.model_dump_json(exclude_none=True)
2751 if isinstance(model_response, BaseModel)
2752 else None
2753 ),
2754 litellm_changed_by=user_api_key_dict.user_id,
2755 litellm_proxy_admin_name=LITELLM_PROXY_ADMIN_NAME,
2756 )
2757 )
2759 raise_if_reload_degraded_serving(
2760 before=live_before_reload,
2761 written_models=[(_model_id, getattr(model_response, "model_info", None))],
2762 action="update",
2763 still_desired=reload_outcome.still_desired,
2764 live_after=reload_outcome.live_after,
2765 )
2767 return model_response
2768 except Exception as e:
2769 verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_model(): Exception occured - %s", e)
2770 if isinstance(e, HTTPException): 2770 ↛ 2771line 2770 didn't jump to line 2771 because the condition on line 2770 was never true
2771 raise ProxyException(
2772 message=getattr(e, "detail", f"Authentication Error({e})"),
2773 type=ProxyErrorTypes.auth_error,
2774 param=getattr(e, "param", "None"),
2775 code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
2776 )
2777 elif isinstance(e, ProxyException): 2777 ↛ 2778line 2777 didn't jump to line 2778 because the condition on line 2777 was never true
2778 raise e
2779 raise ProxyException(
2780 message="Authentication Error, " + str(e),
2781 type=ProxyErrorTypes.auth_error,
2782 param=getattr(e, "param", "None"),
2783 code=status.HTTP_400_BAD_REQUEST,
2784 )
2787@router.post(
2788 "/model_group/make_public",
2789 description="Update which model groups are public",
2790 tags=["model management"],
2791 dependencies=[Depends(user_api_key_auth)],
2792)
2793async def update_public_model_groups(
2794 request: UpdatePublicModelGroupsRequest,
2795 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
2796):
2797 """
2798 Update which model groups are public.
2800 This endpoint allows admins to specify which model groups should be publicly accessible.
2801 Public model groups are visible via the /public/model_hub endpoint.
2803 Args:
2804 request: Request containing list of model group names to make public
2805 user_api_key_dict: User authentication information
2807 Returns:
2808 Success message with updated public model groups
2810 Raises:
2811 ProxyException: For various error conditions including authentication errors
2812 """
2813 try:
2814 # Update the public model groups
2815 from litellm.proxy.proxy_server import proxy_config, store_model_in_db
2817 # Check if user has admin permissions
2818 if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: 2818 ↛ 2819line 2818 didn't jump to line 2819 because the condition on line 2818 was never true
2819 raise HTTPException(
2820 status_code=403,
2821 detail={
2822 "error": f"Only proxy admins can update public model groups. Your role={user_api_key_dict.user_role}"
2823 },
2824 )
2826 # Check if STORE_MODEL_IN_DB is enabled
2827 if store_model_in_db is not True: 2827 ↛ 2828line 2827 didn't jump to line 2828 because the condition on line 2827 was never true
2828 raise HTTPException(
2829 status_code=500,
2830 detail={"error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."},
2831 )
2833 # Load existing config first (this may overwrite in-memory litellm settings
2834 # from DB values via _update_config_from_db), so set the in-memory value AFTER
2835 config: Final = await proxy_config.get_config()
2837 # Update config with new settings
2838 if "litellm_settings" not in config:
2839 config["litellm_settings"] = {}
2841 config["litellm_settings"]["public_model_groups"] = request.model_groups
2843 # Save the updated config
2844 await proxy_config.save_config(new_config=config)
2846 # Set in-memory value AFTER get_config() and save_config() to avoid
2847 # get_config() overwriting with stale DB value
2848 litellm.public_model_groups = request.model_groups
2850 verbose_proxy_logger.debug(
2851 "Updated public model groups to: %s by user: %s", request.model_groups, user_api_key_dict.user_id
2852 )
2854 return {
2855 "message": "Successfully updated public model groups",
2856 "public_model_groups": request.model_groups,
2857 "updated_by": user_api_key_dict.user_id,
2858 }
2860 except Exception as e:
2861 verbose_proxy_logger.exception("Error updating public model groups: %s", e)
2863 if isinstance(e, HTTPException): 2863 ↛ 2864line 2863 didn't jump to line 2864 because the condition on line 2863 was never true
2864 raise e
2866 raise ProxyException(
2867 message=f"Error updating public model groups: {e}",
2868 type=ProxyErrorTypes.internal_server_error,
2869 code=status.HTTP_500_INTERNAL_SERVER_ERROR,
2870 param=None,
2871 )
2874@router.post(
2875 "/model_hub/update_useful_links",
2876 description="Update useful links",
2877 tags=["model management"],
2878 dependencies=[Depends(user_api_key_auth)],
2879)
2880async def update_useful_links(
2881 request: UpdateUsefulLinksRequest,
2882 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
2883):
2884 """
2885 Update useful links.
2886 """
2887 try:
2888 # Update the public model groups
2889 from litellm.proxy.proxy_server import proxy_config
2891 # Check if user has admin permissions
2892 if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: 2892 ↛ 2893line 2892 didn't jump to line 2893 because the condition on line 2892 was never true
2893 raise HTTPException(
2894 status_code=403,
2895 detail={
2896 "error": f"Only proxy admins can update public model groups. Your role={user_api_key_dict.user_role}"
2897 },
2898 )
2900 # Load existing config first (this may overwrite in-memory litellm settings
2901 # from DB values via _update_config_from_db), so set the in-memory value AFTER
2902 config: Final = await proxy_config.get_config()
2904 # Update config with new settings
2905 if "litellm_settings" not in config: 2905 ↛ 2906line 2905 didn't jump to line 2906 because the condition on line 2905 was never true
2906 config["litellm_settings"] = {}
2908 config["litellm_settings"]["public_model_groups_links"] = request.useful_links
2910 # Save the updated config
2911 await proxy_config.save_config(new_config=config)
2913 # Set in-memory value AFTER get_config() and save_config() to avoid
2914 # get_config() overwriting with stale DB value
2915 litellm.public_model_groups_links = request.useful_links
2917 verbose_proxy_logger.debug(
2918 "Updated useful links to: %s by user: %s", request.useful_links, user_api_key_dict.user_id
2919 )
2921 return {
2922 "message": "Successfully updated useful links",
2923 "useful_links": request.useful_links,
2924 "updated_by": user_api_key_dict.user_id,
2925 }
2927 except Exception as e:
2928 verbose_proxy_logger.exception("Error updating public model groups: %s", e)
2930 if isinstance(e, HTTPException): 2930 ↛ 2931line 2930 didn't jump to line 2931 because the condition on line 2930 was never true
2931 raise e
2933 raise ProxyException(
2934 message=f"Error updating public model groups: {e}",
2935 type=ProxyErrorTypes.internal_server_error,
2936 code=status.HTTP_500_INTERNAL_SERVER_ERROR,
2937 param=None,
2938 )
2941def _validated_labeled_tiers(
2942 tier_labels: dict[ComplexityTier, str], # mutable-ok: Pydantic materializes JSON object fields as dicts
2943) -> tuple[tuple[ComplexityTier, str], ...]:
2944 """Validate tier labels once for both prompt-preview transports."""
2945 try:
2946 return ComplexityRouterConfig(tier_labels=tier_labels).labeled_tiers()
2947 except (TypeError, ValidationError) as e:
2948 raise ProxyException(
2949 message=f"tier_labels must be a JSON object of tier name to display name: {e}",
2950 type=ProxyErrorTypes.bad_request_error,
2951 code=status.HTTP_400_BAD_REQUEST,
2952 param="tier_labels",
2953 ) from e
2956def _labeled_tiers_from_query(tier_labels: str | None) -> tuple[tuple[ComplexityTier, str], ...] | None:
2957 """Resolve the tier_labels query param into the labeled tiers the rubric is built from."""
2958 if not tier_labels:
2959 return None
2960 try:
2961 parsed: Final = json.loads(tier_labels)
2962 except JSONDecodeError as e:
2963 raise ProxyException(
2964 message=f"tier_labels must be a JSON object of tier name to display name: {e}",
2965 type=ProxyErrorTypes.bad_request_error,
2966 code=status.HTTP_400_BAD_REQUEST,
2967 param="tier_labels",
2968 ) from e
2969 return _validated_labeled_tiers(parsed)
2972class AutoRouterClassifierPromptPreviewRequest(BaseModel):
2973 """A POST rather than query params: the classification sections are the operator's own text,
2974 which must not reach access logs through a URL."""
2976 tier_definitions: tuple[TierDefinition, ...] | None = None
2977 tier_labels: dict[ComplexityTier, str] | None = None # mutable-ok: FastAPI parses JSON object fields into dicts
2978 classification_rubric: ClassificationRubric | None = None
2979 context_window_size: Annotated[int, Field(ge=0)] = DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE
2980 classification_prompt: str | None = None
2981 classification_examples: str | None = None
2983 _normalize_prompt = field_validator("classification_prompt")(normalize_classification_prompt)
2984 _normalize_examples = field_validator("classification_examples")(normalize_classification_examples)
2987@router.post(
2988 "/auto_router/classifier/default_prompt",
2989 description="Get the system prompt an auto-router's LLM classifier sends for an edited tier set",
2990 tags=["model management"], # mutable-ok: fastapi's decorator signature types tags as a list
2991 dependencies=[Depends(user_api_key_auth)], # mutable-ok: fastapi's decorator signature types dependencies as a list
2992)
2993async def preview_auto_router_classifier_prompt(
2994 request: AutoRouterClassifierPromptPreviewRequest,
2995) -> AutoRouterClassifierDefaultPromptResponse:
2996 """
2997 Get the classifier system prompt an edited tier set sends, so the dashboard can show it.
2999 Built by the same function the live classifier uses, so the preview cannot drift from what the
3000 router sends. Payload validity beyond a renderable definition stays the dry-run's job.
3001 """
3002 labeled_tiers: Final = _validated_labeled_tiers(request.tier_labels or {}) # mutable-ok: Pydantic field default
3003 system_prompt: Final = (
3004 custom_tier_classification_prompt(
3005 request.tier_definitions,
3006 request.classification_prompt,
3007 request.context_window_size,
3008 classification_examples=request.classification_examples,
3009 )
3010 if request.tier_definitions is not None
3011 else built_in_tier_classification_prompt(
3012 request.classification_prompt,
3013 request.context_window_size,
3014 labeled_tiers=labeled_tiers,
3015 classification_rubric=request.classification_rubric,
3016 classification_examples=request.classification_examples,
3017 )
3018 )
3019 return AutoRouterClassifierDefaultPromptResponse(system_prompt=system_prompt)
3022@router.get(
3023 "/auto_router/classifier/default_prompt",
3024 description="Get the built-in system prompt used by an auto-router's LLM classifier",
3025 tags=["model management"], # mutable-ok: fastapi's decorator signature types tags as a list
3026 dependencies=[Depends(user_api_key_auth)], # mutable-ok: fastapi's decorator signature types dependencies as a list
3027)
3028async def get_auto_router_classifier_default_prompt(
3029 context_window_size: int = DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
3030 tier_labels: str | None = None,
3031 classification_rubric: ClassificationRubric | None = None,
3032) -> AutoRouterClassifierDefaultPromptResponse:
3033 """
3034 Get the classifier system prompt a router would send, so the dashboard can show it.
3036 The prompt's closing line depends on whether prior conversation turns are quoted to the
3037 classifier, its tier bullets are named by the router's tier_labels, and its calibration examples
3038 come from the router's classification rubric, so the caller passes all three to get the text that router
3039 would actually send rather than a rubric it does not use.
3041 An edited tier set replaces the whole rubric; POST to this path for that prompt, which carries
3042 the operator's own instructions and so must not ride in a query string.
3044 Parameters:
3045 - context_window_size: int - The router's classifier_context_window_size. Defaults to the
3046 built-in default.
3047 - tier_labels: str | None - The router's tier_labels as a JSON object of canonical tier name to
3048 display name, e.g. `{"SIMPLE": "Cheap"}`. Omit or pass an empty object for the default names.
3049 - classification_rubric: ClassificationRubric | None - The router's
3050 classifier_llm_config.classification_rubric. Omit for the default.
3051 """
3052 if context_window_size < 0:
3053 raise ProxyException(
3054 message="context_window_size must be non-negative",
3055 type=ProxyErrorTypes.bad_request_error,
3056 code=status.HTTP_400_BAD_REQUEST,
3057 param="context_window_size",
3058 )
3060 labeled_tiers: Final = _labeled_tiers_from_query(tier_labels)
3061 return AutoRouterClassifierDefaultPromptResponse(
3062 system_prompt=(
3063 classification_system_prompt(context_window_size, classification_rubric=classification_rubric)
3064 if labeled_tiers is None
3065 else classification_system_prompt(
3066 context_window_size, labeled_tiers=labeled_tiers, classification_rubric=classification_rubric
3067 )
3068 )
3069 )
3072def _deduplicate_litellm_router_models(models: list[dict]) -> list[dict]:
3073 """
3074 Deduplicate models based on their model_info.id field.
3075 Returns a list of unique models keeping only the first occurrence of each model ID.
3077 Args:
3078 models: List of model dictionaries containing model_info
3080 Returns:
3081 List of deduplicated model dictionaries
3082 """
3083 seen_ids: Final = set()
3084 unique_models: Final = []
3085 for model in models: 3085 ↛ 3086line 3085 didn't jump to line 3086 because the loop on line 3085 never started
3086 model_id = model.get("model_info", {}).get("id", None)
3087 if model_id is not None and model_id not in seen_ids:
3088 unique_models.append(model)
3089 seen_ids.add(model_id)
3090 return unique_models
3093def model_info_as_mapping(model_info: object) -> Mapping[str, object] | None:
3094 """A DB row's model_info column arrives as a dict or as its JSON string depending on
3095 the query path, and every consumer needs the mapping. Single owner of that parse:
3096 returns None when no usable mapping exists (None, an unparseable string, or JSON
3097 that is not an object), and callers choose what None means for them."""
3098 if isinstance(model_info, Mapping): 3098 ↛ 3100line 3098 didn't jump to line 3100 because the condition on line 3098 was always true
3099 return model_info
3100 if not isinstance(model_info, str):
3101 return None
3102 try:
3103 parsed: Final = json.loads(model_info)
3104 except (TypeError, ValueError):
3105 return None
3106 return parsed if isinstance(parsed, Mapping) else None
3109def _expects_liveness_on_this_pod(model_info: object) -> bool:
3110 from litellm.router import model_info_is_active_for_environment
3112 try:
3113 return model_info_is_active_for_environment(model_info=model_info_as_mapping(model_info))
3114 except ValueError:
3115 return True
3118def live_model_ids_snapshot() -> frozenset[str]:
3119 """The ids this pod's router is currently serving, read fresh from the module global
3120 because a reload can rebind it. The empirical ground truth every verdict below is
3121 computed from; an absent router serves nothing."""
3122 from litellm.proxy.proxy_server import llm_router
3124 if llm_router is None: 3124 ↛ 3125line 3124 didn't jump to line 3125 because the condition on line 3124 was never true
3125 return frozenset()
3126 return frozenset(llm_router.get_model_ids())
3129def reload_serving_verdict(
3130 before: frozenset[str],
3131 written_models: Sequence[tuple[str, object]],
3132 written_must_serve: bool,
3133 still_desired: frozenset[str] | None = None,
3134 live_after: frozenset[str] | None = None,
3135) -> tuple[tuple[str, ...], tuple[str, ...]]:
3136 """Judge a write-triggered reload by diffing the router's serving state instead of
3137 trusting any layer of the reload stack to report its own failure.
3139 The full cell matrix, per id:
3140 - written, must-serve (the write's purpose is this model's serving state): live now
3141 is fine; not live is reported unless the row is deliberately inactive for this
3142 pod's LITELLM_ENVIRONMENT; a row whose model_info cannot be read counts as
3143 expecting to serve, so its drop is still reported
3144 - written, metadata-only (must_not_degrade): live before and gone now is reported;
3145 a row that was already not serving stays silent, because its deadness predates
3146 this write and blaming it would block unrelated metadata fixes
3147 - not written but live before and gone now: collateral degradation of this pod
3148 caused by the reload this request triggered (a wholesale re-add failure, or a
3149 newly introduced conflict), reported only when the db still wants that id
3151 ``still_desired`` is the db + config id set the reload just reconciled against. An
3152 id absent from it was deleted on purpose, most often by another pod this one had not
3153 yet polled, so the reload dropping it is the reconcile working rather than damage.
3154 Without it (no reconcile ran) every drop is reported, which is the safe direction.
3156 ``live_after`` is the router's serving state captured by the reload itself, while it
3157 still held MODEL_RECONCILE_LOCK. Pass it whenever the caller has it: re-reading the
3158 router here instead means sampling it after the lock was released, where the NEXT
3159 reconcile's leading wipe (clear_cache un-serves every db model before reloading
3160 them) shows up as this reload having dropped them. Falling back to a fresh read is
3161 only correct when no reconcile ran and there is nothing to be concurrent with.
3163 Returns (written ids violating their obligation, collateral ids no longer served).
3164 """
3165 now: Final = live_model_ids_snapshot() if live_after is None else live_after
3166 written_ids: Final = frozenset(model_id for model_id, _ in written_models)
3167 if written_must_serve: 3167 ↛ 3174line 3167 didn't jump to line 3174 because the condition on line 3167 was always true
3168 missing = tuple(
3169 model_id
3170 for model_id, model_info in written_models
3171 if model_id not in now and _expects_liveness_on_this_pod(model_info)
3172 )
3173 else:
3174 missing = tuple(model_id for model_id, _ in written_models if model_id in before and model_id not in now)
3175 dropped: Final = before - now - written_ids
3176 collateral: Final = tuple(sorted(dropped if still_desired is None else dropped & still_desired))
3177 return (missing, collateral)
3180def raise_if_reload_degraded_serving(
3181 before: frozenset[str],
3182 written_models: Sequence[tuple[str, object]],
3183 action: str,
3184 still_desired: frozenset[str] | None = None,
3185 live_after: frozenset[str] | None = None,
3186) -> None:
3187 """The caller-visible error this pod's model-write endpoints owe their caller when
3188 the model they wrote is not being served after the reload they triggered. The DB
3189 write is durable either way and every other pod reloads on its own interval; this
3190 speaks only for the handling pod.
3192 Callers hold a ReconcileOutcome from the reload; pass BOTH of its fields. Supplying
3193 still_desired without live_after mixes a snapshot taken under the reconcile lock
3194 with one taken after it was released, which is what makes a concurrent model write
3195 look like collateral damage."""
3196 missing, collateral = reload_serving_verdict(
3197 before=before,
3198 written_models=written_models,
3199 written_must_serve=True,
3200 still_desired=still_desired,
3201 live_after=live_after,
3202 )
3203 if not missing and not collateral: 3203 ↛ 3204line 3203 didn't jump to line 3204 because the condition on line 3203 was never true
3204 return
3205 missing_clause: Final = (
3206 f"the model id(s) {list(missing)} are not live in this pod's router after the reload and are not "
3207 "being served by this pod."
3208 if missing
3209 else "the reload it triggered degraded this pod's serving state."
3210 )
3211 collateral_clause: Final = (
3212 f" Previously served model id(s) {list(collateral)} are also no longer being served by this pod."
3213 if collateral
3214 else ""
3215 )
3216 raise ProxyException(
3217 message=(
3218 f"Model {action} was saved to the database, but {missing_clause}{collateral_clause} "
3219 "Other pods reload on their own interval. Check server logs for 'Error upserting deployment' or "
3220 "'Error creating deployment' for the cause."
3221 ),
3222 type=ProxyErrorTypes.internal_server_error,
3223 code=status.HTTP_500_INTERNAL_SERVER_ERROR,
3224 param=None,
3225 )
3228async def clear_cache() -> ReconcileOutcome:
3229 """
3230 Clear router caches and reload models.
3232 Returns what the reload saw (see ReconcileOutcome) so callers can pass it to
3233 raise_if_reload_degraded_serving.
3235 Runs under MODEL_RECONCILE_LOCK for its whole extent, not just the reload at the
3236 end, so the auto-router reset and the reload that rebuilds those routers are atomic
3237 to any other reconcile. The inner call is _add_deployment_locked because
3238 add_deployment would re-acquire the same non-reentrant lock and deadlock.
3239 """
3240 from litellm.proxy.proxy_server import (
3241 MODEL_RECONCILE_LOCK,
3242 llm_router,
3243 prisma_client,
3244 proxy_config,
3245 proxy_logging_obj,
3246 verbose_proxy_logger,
3247 )
3249 if llm_router is None or prisma_client is None:
3250 verbose_proxy_logger.debug("llm_router or prisma_client is None, skipping cache clear")
3251 return ReconcileOutcome(still_desired=None, live_after=None)
3253 async with MODEL_RECONCILE_LOCK:
3254 try:
3255 # Only clear DB models, preserve config models
3256 verbose_proxy_logger.debug("Clearing only DB models, preserving config models")
3258 # Get current models and filter out DB models
3259 current_models: Final = llm_router.model_list.copy()
3260 config_models: Final = []
3261 db_model_ids: Final = []
3263 db_router_names: Final = set()
3265 for model in current_models:
3266 model_info = model.get("model_info", {})
3267 if model_info.get("db_model", False):
3268 db_model_ids.append(model_info.get("id"))
3269 # Auto-router deployments (and only those) are wiped here, in the
3270 # same pass, so the reload rebuilds them -- see the comment below.
3271 model_name = model.get("model_name")
3272 if model_name is not None and str(model.get("litellm_params", {}).get("model", "")).startswith(
3273 "auto_router/"
3274 ):
3275 db_router_names.add(model_name)
3276 router_model_id = model_info.get("id")
3277 if router_model_id is not None:
3278 llm_router.delete_deployment(id=router_model_id)
3279 else:
3280 # This is a config model, preserved by the reconcile below
3281 config_models.append(model)
3283 # ORDINARY db deployments are deliberately NOT wiped. This used to
3284 # delete_deployment() every db model before the reload put them back, which
3285 # left the router serving ZERO db models for the whole width of the reload
3286 # -- a real data-plane hole that every inference request landing in it fell
3287 # into. It was also redundant for them: the reload's _delete_deployment
3288 # evicts exactly the ids the db no longer lists, and upsert_deployment
3289 # pops-and-re-adds a deployment whose params changed while no-opping one
3290 # that did not, so the reconcile converges on its own. Every mutation is
3291 # visible to that comparison -- `blocked` and (for premium) `updated_at`
3292 # are written into model_info.
3293 #
3294 # AUTO-ROUTER db deployments are the exception and ARE wiped -- in the
3295 # classification pass above, together with the strategy entries popped
3296 # just below. Their strategy registries are keyed
3297 # by model_name, which no deployment-id reconcile touches, so they have to
3298 # be popped and rebuilt here. But the rebuild only happens on the ADD path:
3299 # Router.upsert_deployment returns early when a deployment is unchanged and
3300 # never reaches add_deployment -> _add_deployment ->
3301 # init_auto_router_deployment, which is what repopulates the registries.
3302 # Popping without deleting would therefore strip every db-backed auto,
3303 # complexity, adaptive and quality router on this pod and never put it back,
3304 # so ANY unrelated model write would leave them unroutable until a restart.
3305 # Deleting the deployment forces upsert down the add path, which rebuilds
3306 # both the deployment and its strategy entry.
3307 #
3308 # That pass restricts the wipe to deployments whose model is actually an
3309 # auto_router/* so a config router that merely shares a model_name with a
3310 # regular db model isn't evicted -- config routers are never re-added by the
3311 # reload (it only reloads db models) and would be permanently unroutable.
3312 # The auto_router/ prefix also covers quality_router/ and adaptive_router/,
3313 # so pop the name from every registry (no-op where absent); a missing
3314 # quality/adaptive entry would otherwise make init raise "already exists"
3315 # on reload and abort it.
3316 for model_name in db_router_names:
3317 llm_router.auto_routers.pop(model_name, None)
3318 llm_router.complexity_routers.pop(model_name, None)
3319 llm_router.adaptive_routers.pop(model_name, None)
3320 llm_router.quality_routers.pop(model_name, None)
3322 # Reload only DB models. _add_deployment_locked, not add_deployment: this
3323 # coroutine already holds MODEL_RECONCILE_LOCK and asyncio.Lock is not
3324 # reentrant, so the public wrapper would deadlock against itself.
3325 outcome: Final = await proxy_config._add_deployment_locked(
3326 prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
3327 )
3329 verbose_proxy_logger.debug(
3330 "Reconciled %s DB models, preserved %s config models", len(db_model_ids), len(config_models)
3331 )
3332 return outcome
3333 except Exception as e:
3334 verbose_proxy_logger.exception("Failed to clear cache and reload models. Due to error - %s", e)
3335 return ReconcileOutcome(still_desired=None, live_after=None)