Coverage for .venv/lib/python3.13/site-packages/litellm/proxy/management_helpers/bulk_user_deletion.py: 61%

259 statements  

« prev     ^ index     » next       coverage.py v7.15.2, created at 2026-10-10 12:01 +0000

1"""Batched deletes behind `POST /management/v1/users/bulk_delete` and 

2`POST /management/v1/teams/{team_id}/members/bulk_delete`. 

3 

4Each team a batch touches is rewritten exactly once, under the same advisory lock 

5`/team/member_delete` takes and from a roster re-read under that lock, so a concurrent 

6member_add on the team is never overwritten from a stale read. A user batch runs in one 

7transaction, taking its team locks in sorted order, so either every team rewrite and every 

8user row delete lands or none of them does. 

9""" 

10 

11import asyncio 

12import json 

13from collections.abc import Awaitable, Iterable, Mapping, Sequence 

14from dataclasses import dataclass 

15from datetime import timedelta 

16from types import MappingProxyType 

17from typing import TYPE_CHECKING, Final 

18 

19from fastapi import HTTPException 

20from typing_extensions import ReadOnly, TypedDict 

21 

22from litellm._logging import verbose_proxy_logger 

23from litellm.integrations.prometheus import PrometheusLogger 

24from litellm.proxy._types import ( 

25 LiteLLM_TeamTable, 

26 LitellmUserRoles, 

27 Member, 

28 MemberDeleteRequest, 

29 UserAPIKeyAuth, 

30) 

31from litellm.proxy.auth.auth_checks import delete_cache_key_objects, get_jwt_key_mapping_cache_keys_for_tokens 

32from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast 

33from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache 

34from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks 

35from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks 

36from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem 

37from litellm.proxy.management_endpoints.common_utils import ( 

38 _is_user_org_admin_for_team, # pyright: ignore[reportPrivateUsage] # same check /team/member_delete uses 

39 _is_user_team_admin, # pyright: ignore[reportPrivateUsage] # same check /team/member_delete uses 

40) 

41from litellm.proxy.management_endpoints.key_management_endpoints import ( 

42 _persist_deleted_verification_tokens, # pyright: ignore[reportPrivateUsage] # same audit path /key/delete uses 

43) 

44from litellm.proxy.management_helpers.access_group_team_sync import TEAM_ADVISORY_LOCK_SQL 

45from litellm.proxy.utils import PrismaClient, ProxyLogging 

46from litellm.repositories.table_repositories import ( 

47 OrganizationMembershipRepository, 

48 TeamMembershipRepository, 

49) 

50from litellm.repositories.team_repository import TeamRepository 

51from litellm.repositories.user_repository import UserRepository 

52from litellm.types.proxy.management_endpoints.internal_user_endpoints import ( 

53 BulkDeleteUserRequest, 

54 UserDeleteResult, 

55) 

56from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail 

57from litellm.types.proxy.management_endpoints.team_endpoints import ( 

58 BulkTeamMemberDeleteRequest, 

59 TeamMemberDeleteResult, 

60) 

61 

62if TYPE_CHECKING: 62 ↛ 63line 62 didn't jump to line 63 because the condition on line 62 was never true

63 from prisma import Prisma 

64 from prisma import models as prisma_models 

65 

66 from litellm.repositories.prisma_protocols import TableActions 

67 

68_AUDIT_LOG_CONCURRENCY: Final = 10 

69_BATCH_TX_TIMEOUT: Final = timedelta(seconds=60) 

70 

71 

72class _OrgAdminFilter(TypedDict): 

73 user_id: ReadOnly[str] 

74 user_role: ReadOnly[str] 

75 

76 

77class _RosterData(TypedDict): 

78 members_with_roles: ReadOnly[str] 

79 

80 

81class _TeamsSet(TypedDict): 

82 set: ReadOnly[tuple[str, ...]] 

83 

84 

85class _TeamsData(TypedDict): 

86 teams: ReadOnly[_TeamsSet] 

87 

88 

89@dataclass(frozen=True, slots=True) 

90class _TeamRemoval: 

91 """One team's rewrite. `removed` holds the user ids taken off the team (roster, `teams` array, or both); 

92 `matched` holds the indexes into the requested members that named at least one of them.""" 

93 

94 team: LiteLLM_TeamTable 

95 removed: frozenset[str] 

96 matched: frozenset[int] 

97 deleted_keys: tuple["prisma_models.LiteLLM_VerificationToken", ...] 

98 jwt_mapping_cache_keys: tuple[str, ...] 

99 

100 @property 

101 def deleted_key_tokens(self) -> tuple[str, ...]: 

102 return tuple(k.token for k in self.deleted_keys) 

103 

104 

105@dataclass(frozen=True, slots=True) 

106class _UserBatchDeletion: 

107 removals: Mapping[str, _TeamRemoval] 

108 deleted_keys: tuple["prisma_models.LiteLLM_VerificationToken", ...] 

109 jwt_mapping_cache_keys: tuple[str, ...] 

110 

111 @property 

112 def deleted_key_tokens(self) -> tuple[str, ...]: 

113 return tuple(k.token for k in self.deleted_keys) 

114 

115 

116@dataclass(frozen=True, slots=True) 

117class _DeletedKeys: 

118 keys: tuple["prisma_models.LiteLLM_VerificationToken", ...] 

119 jwt_mapping_cache_keys: tuple[str, ...] 

120 

121 

122def _team_not_found(team_id: str) -> ManagementProblem: 

123 return ManagementProblem( 

124 ProblemDetail( 

125 type=f"{PROBLEM_TYPE_BASE}team-not-found", 

126 title="Team not found", 

127 status=404, 

128 detail=f"Team id={team_id} does not exist in db", 

129 ) 

130 ) 

131 

132 

133def _forbidden(detail: str) -> ManagementProblem: 

134 return ManagementProblem( 

135 ProblemDetail(type=f"{PROBLEM_TYPE_BASE}forbidden", title="Forbidden", status=403, detail=detail) 

136 ) 

137 

138 

139def _in_filter(field: str, values: Iterable[str]) -> Mapping[str, object]: 

140 return {field: {"in": sorted(values)}} # mutable-ok: Prisma query filters are dict-shaped 

141 

142 

143def _eq_filter(field: str, value: str) -> Mapping[str, object]: 

144 return {field: value} # mutable-ok: Prisma query filters are dict-shaped 

145 

146 

147def _team_users_filter(team_id: str, user_ids: Iterable[str]) -> Mapping[str, object]: 

148 return {"team_id": team_id, **_in_filter("user_id", user_ids)} # mutable-ok: Prisma query filters are dict-shaped 

149 

150 

151def _any_filter(*clauses: Mapping[str, object]) -> Mapping[str, object]: 

152 return {"OR": clauses} # mutable-ok: Prisma query filters are dict-shaped 

153 

154 

155def _team_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamTable]": 

156 return tx.litellm_teamtable # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do 

157 

158 

159def _user_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_UserTable]": 

160 return tx.litellm_usertable # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do 

161 

162 

163def _membership_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamMembership]": 

164 return tx.litellm_teammembership # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do 

165 

166 

167def _token_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_VerificationToken]": 

168 return tx.litellm_verificationtoken # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do 

169 

170 

171def _invitation_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_InvitationLink]": 

172 return tx.litellm_invitationlink # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do 

173 

174 

175def _org_membership_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_OrganizationMembership]": 

176 return tx.litellm_organizationmembership # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do 

177 

178 

179def _same_email(email: str | None, request: MemberDeleteRequest) -> bool: 

180 return request.user_email is not None and request.user_email == email 

181 

182 

183def _addresses_member(member: Member, request: MemberDeleteRequest) -> bool: 

184 if request.user_id is None: 

185 return _same_email(member.user_email, request) 

186 return request.user_id == member.user_id or (member.user_id is None and _same_email(member.user_email, request)) 

187 

188 

189def _with_row_email(request: MemberDeleteRequest, email_of: Mapping[str, str]) -> MemberDeleteRequest: 

190 if request.user_id is None or request.user_email is not None: 

191 return request 

192 return MemberDeleteRequest(user_id=request.user_id, user_email=email_of.get(request.user_id)) 

193 

194 

195def _addresses_user(user: "prisma_models.LiteLLM_UserTable", request: MemberDeleteRequest) -> bool: 

196 if request.user_id is None: 

197 return _same_email(user.user_email, request) 

198 return request.user_id == user.user_id 

199 

200 

201def _error_message(exc: BaseException) -> str: 

202 if isinstance(exc, ManagementProblem): 

203 return exc.problem.detail 

204 if isinstance(exc, HTTPException) and isinstance(exc.detail, dict): 

205 return str(exc.detail.get("error", exc.detail)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # HTTPException.detail is untyped 

206 if isinstance(exc, HTTPException): 

207 return str(exc.detail) 

208 return str(exc) or type(exc).__name__ 

209 

210 

211async def _bounded(awaitables: Iterable[Awaitable[object]]) -> tuple[object | BaseException, ...]: 

212 semaphore: Final = asyncio.Semaphore(_AUDIT_LOG_CONCURRENCY) 

213 

214 async def run(awaitable: Awaitable[object]) -> object: 

215 async with semaphore: 

216 return await awaitable 

217 

218 return tuple(await asyncio.gather(*(run(a) for a in awaitables), return_exceptions=True)) 

219 

220 

221async def _remove_members_from_team( 

222 prisma_client: PrismaClient, 

223 tx: "Prisma", 

224 team_id: str, 

225 members: Sequence[MemberDeleteRequest], 

226 user_api_key_dict: UserAPIKeyAuth, 

227) -> _TeamRemoval: 

228 await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id) 

229 roster: Final = await TeamRepository(prisma_client).get_members_with_roles_locked(tx, team_id) 

230 if roster is None: 230 ↛ 231line 230 didn't jump to line 231 because the condition on line 230 was never true

231 raise _team_not_found(team_id) 

232 

233 requested_ids: Final = frozenset(r.user_id for r in members if r.user_id is not None) 

234 requested_emails: Final = frozenset(r.user_email for r in members if r.user_id is None and r.user_email) 

235 requested_rows: Final = await _user_tx_db(tx).find_many( 

236 where=_any_filter(_in_filter("user_id", requested_ids), _in_filter("user_email", requested_emails)) 

237 ) 

238 email_of: Final = MappingProxyType( 

239 {u.user_id: u.user_email for u in requested_rows if u.user_email is not None and team_id in u.teams} 

240 ) 

241 requests: Final = tuple(_with_row_email(r, email_of) for r in members) 

242 removed_members: Final = tuple(m for m in roster if any(_addresses_member(m, r) for r in requests)) 

243 kept_members: Final = tuple(m for m in roster if not any(_addresses_member(m, r) for r in requests)) 

244 removed_ids: Final = frozenset(m.user_id for m in removed_members if m.user_id is not None) 

245 unfetched_ids: Final = removed_ids - frozenset(u.user_id for u in requested_rows) 

246 removed_rows: Final = ( 

247 await _user_tx_db(tx).find_many(where=_in_filter("user_id", unfetched_ids)) if unfetched_ids else () 

248 ) 

249 stale_rows: Final = tuple(u for u in (*requested_rows, *removed_rows) if team_id in u.teams) 

250 cleanup_ids: Final = removed_ids | frozenset(u.user_id for u in stale_rows) 

251 matched: Final = frozenset( 

252 i 

253 for i, r in enumerate(requests) 

254 if any(_addresses_member(m, r) for m in removed_members) or any(_addresses_user(u, r) for u in stale_rows) 

255 ) 

256 keys: Final = await _token_tx_db(tx).find_many(where=_team_users_filter(team_id, cleanup_ids)) 

257 jwt_mapping_cache_keys: Final = await get_jwt_key_mapping_cache_keys_for_tokens( 

258 hashed_tokens=tuple(k.token for k in keys), 

259 prisma_client=prisma_client, 

260 ) 

261 

262 if removed_members: 262 ↛ 263line 262 didn't jump to line 263 because the condition on line 262 was never true

263 roster_data: Final[_RosterData] = { 

264 "members_with_roles": json.dumps(tuple(m.model_dump() for m in kept_members)) 

265 } 

266 await _team_tx_db(tx).update(where=_eq_filter("team_id", team_id), data=roster_data) 

267 for row in stale_rows: 267 ↛ 268line 267 didn't jump to line 268 because the loop on line 267 never started

268 teams_data: _TeamsData = {"teams": {"set": tuple(t for t in row.teams if t != team_id)}} 

269 await _user_tx_db(tx).update(where=_eq_filter("user_id", row.user_id), data=teams_data) 

270 await _membership_tx_db(tx).delete_many(where=_team_users_filter(team_id, cleanup_ids)) 

271 if keys: 271 ↛ 272line 271 didn't jump to line 272 because the condition on line 271 was never true

272 await _persist_deleted_verification_tokens( 

273 keys=keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken 

274 prisma_client=prisma_client, 

275 user_api_key_dict=user_api_key_dict, 

276 litellm_changed_by=None, 

277 tx=tx, 

278 ) 

279 await _token_tx_db(tx).delete_many(where=_team_users_filter(team_id, cleanup_ids)) 

280 

281 return _TeamRemoval( 

282 team=LiteLLM_TeamTable( 

283 team_id=team_id, 

284 members_with_roles=kept_members, # pyright: ignore[reportArgumentType] # pydantic coerces the tuple into the list field 

285 ), 

286 removed=cleanup_ids, 

287 matched=matched, 

288 deleted_keys=tuple(keys), 

289 jwt_mapping_cache_keys=jwt_mapping_cache_keys, 

290 ) 

291 

292 

293def _emit_team_members_metric(team: LiteLLM_TeamTable) -> None: 

294 prometheus_logger: Final = PrometheusLogger.get_instance() 

295 if prometheus_logger is None: 295 ↛ 297line 295 didn't jump to line 297 because the condition on line 295 was always true

296 return 

297 try: 

298 prometheus_logger.set_team_members_metric(team) 

299 except Exception as e: 

300 verbose_proxy_logger.debug("Prometheus: failed to emit team members metric: %s", str(e)) 

301 

302 

303def _duplicate_member_indexes(members: Sequence[MemberDeleteRequest]) -> frozenset[int]: 

304 return frozenset( 

305 i 

306 for i, m in enumerate(members) 

307 if any( 

308 (m.user_id is not None and m.user_id == earlier.user_id) 

309 or (m.user_email is not None and m.user_email == earlier.user_email) 

310 for earlier in members[:i] 

311 ) 

312 ) 

313 

314 

315async def bulk_remove_team_members( 

316 team_id: str, 

317 data: BulkTeamMemberDeleteRequest, 

318 user_api_key_dict: UserAPIKeyAuth, 

319 prisma_client: PrismaClient, 

320 user_api_key_cache: UserApiKeyCache, 

321 proxy_logging_obj: ProxyLogging | None, 

322) -> tuple[TeamMemberDeleteResult, ...]: 

323 team: Final = await TeamRepository(prisma_client).find_by_id(team_id) 

324 if team is None: 

325 raise _team_not_found(team_id) 

326 

327 if ( 327 ↛ 332line 327 didn't jump to line 332 because the condition on line 327 was never true

328 user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value 

329 and not _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team) 

330 and not await _is_user_org_admin_for_team(user_api_key_dict=user_api_key_dict, team_obj=team) 

331 ): 

332 raise _forbidden( 

333 "Call not allowed. User not proxy admin OR team admin OR org admin for this team. " 

334 f"route='/management/v1/teams/{team_id}/members/bulk_delete'" 

335 ) 

336 

337 duplicates: Final = _duplicate_member_indexes(data.members) 

338 kept_indexes: Final = tuple(i for i in range(len(data.members)) if i not in duplicates) 

339 members: Final = tuple(data.members[i] for i in kept_indexes) 

340 async with prisma_client.tx(timeout=_BATCH_TX_TIMEOUT) as tx: 

341 removal: Final = await _remove_members_from_team(prisma_client, tx, team_id, members, user_api_key_dict) 

342 if removal.deleted_keys: 342 ↛ 343line 342 didn't jump to line 343 because the condition on line 342 was never true

343 KeyManagementEventHooks.create_key_deleted_audit_logs( 

344 keys_being_deleted=removal.deleted_keys, 

345 user_api_key_dict=user_api_key_dict, 

346 litellm_changed_by=None, 

347 ) 

348 await delete_cache_key_objects( 

349 hashed_tokens=removal.deleted_key_tokens, 

350 user_api_key_cache=user_api_key_cache, 

351 proxy_logging_obj=proxy_logging_obj, 

352 ) 

353 await evict_and_broadcast(cache_keys=removal.jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache) 

354 _emit_team_members_metric(removal.team) 

355 

356 matched: Final = frozenset(kept_indexes[j] for j in removal.matched) 

357 

358 def error(index: int) -> str | None: 

359 if index in duplicates: 359 ↛ 360line 359 didn't jump to line 360 because the condition on line 359 was never true

360 return "Duplicate member in request" 

361 return None if index in matched else "User not found in team" 

362 

363 return tuple( 

364 TeamMemberDeleteResult( 

365 user_id=member.user_id, 

366 user_email=member.user_email, 

367 success=i in matched, 

368 error=error(i), 

369 ) 

370 for i, member in enumerate(data.members) 

371 ) 

372 

373 

374async def _caller_admin_org_ids(prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]: 

375 if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value or not user_api_key_dict.user_id: 375 ↛ 377line 375 didn't jump to line 377 because the condition on line 375 was always true

376 return frozenset() 

377 where: Final[_OrgAdminFilter] = { 

378 "user_id": user_api_key_dict.user_id, 

379 "user_role": LitellmUserRoles.ORG_ADMIN.value, 

380 } 

381 memberships: Final = await OrganizationMembershipRepository(prisma_client).table.find_many(where=where) 

382 return frozenset(m.organization_id for m in memberships if m.organization_id) 

383 

384 

385def _scope_error(user_id: str, target_org_ids: frozenset[str], caller_admin_org_ids: frozenset[str]) -> str | None: 

386 if target_org_ids and target_org_ids <= caller_admin_org_ids: 

387 return None 

388 return ( 

389 f"User {user_id} is not within your admin scope. " 

390 "Only PROXY_ADMIN may delete users outside your administered organizations." 

391 ) 

392 

393 

394async def _delete_user_rows( 

395 prisma_client: PrismaClient, 

396 tx: "Prisma", 

397 user_ids: frozenset[str], 

398 user_api_key_dict: UserAPIKeyAuth, 

399 litellm_changed_by: str | None, 

400) -> _DeletedKeys: 

401 keys: Final = await _token_tx_db(tx).find_many(where=_in_filter("user_id", user_ids)) 

402 jwt_mapping_cache_keys: Final = await get_jwt_key_mapping_cache_keys_for_tokens( 

403 hashed_tokens=tuple(k.token for k in keys), 

404 prisma_client=prisma_client, 

405 ) 

406 if keys: 

407 await _persist_deleted_verification_tokens( 

408 keys=keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken 

409 prisma_client=prisma_client, 

410 user_api_key_dict=user_api_key_dict, 

411 litellm_changed_by=litellm_changed_by, 

412 tx=tx, 

413 ) 

414 await _token_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) 

415 await _invitation_tx_db(tx).delete_many( 

416 where=_any_filter( 

417 _in_filter("user_id", user_ids), 

418 _in_filter("created_by", user_ids), 

419 _in_filter("updated_by", user_ids), 

420 ) 

421 ) 

422 await _org_membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) 

423 await _membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) 

424 await _user_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) 

425 return _DeletedKeys(keys=tuple(keys), jwt_mapping_cache_keys=jwt_mapping_cache_keys) 

426 

427 

428async def _delete_users_tx( 

429 prisma_client: PrismaClient, 

430 users: Sequence["prisma_models.LiteLLM_UserTable"], 

431 teams_of: Mapping[str, frozenset[str]], 

432 user_api_key_dict: UserAPIKeyAuth, 

433 litellm_changed_by: str | None, 

434) -> _UserBatchDeletion: 

435 """Rewrites every team the users belong to and deletes their rows in one transaction, so a 

436 failure anywhere rolls back the whole batch. Teams a user still names but which no longer exist 

437 are skipped; the user row goes away regardless.""" 

438 async with prisma_client.tx(timeout=_BATCH_TX_TIMEOUT) as tx: 

439 team_rows: Final = await _team_tx_db(tx).find_many( 

440 where=_in_filter("team_id", frozenset(t for teams in teams_of.values() for t in teams)) 

441 ) 

442 team_ids: Final = tuple(sorted(t.team_id for t in team_rows)) 

443 removals: Final = MappingProxyType( 

444 { 

445 tid: await _remove_members_from_team( 

446 prisma_client, 

447 tx, 

448 tid, 

449 tuple( 

450 MemberDeleteRequest(user_id=u.user_id, user_email=u.user_email) 

451 for u in users 

452 if tid in teams_of[u.user_id] 

453 ), 

454 user_api_key_dict, 

455 ) 

456 for tid in team_ids 

457 } 

458 ) 

459 deleted_keys: Final = await _delete_user_rows( 

460 prisma_client, tx, frozenset(u.user_id for u in users), user_api_key_dict, litellm_changed_by 

461 ) 

462 return _UserBatchDeletion( 

463 removals=removals, 

464 deleted_keys=deleted_keys.keys + tuple(k for r in removals.values() for k in r.deleted_keys), 

465 jwt_mapping_cache_keys=deleted_keys.jwt_mapping_cache_keys 

466 + tuple(k for r in removals.values() for k in r.jwt_mapping_cache_keys), 

467 ) 

468 

469 

470async def _delete_users( 

471 prisma_client: PrismaClient, 

472 users: Sequence["prisma_models.LiteLLM_UserTable"], 

473 teams_of: Mapping[str, frozenset[str]], 

474 user_api_key_dict: UserAPIKeyAuth, 

475 user_api_key_cache: UserApiKeyCache, 

476 proxy_logging_obj: ProxyLogging | None, 

477 litellm_proxy_admin_name: str | None, 

478 litellm_changed_by: str | None, 

479) -> _UserBatchDeletion | str: 

480 """Returns the error message when the transaction rolled back, in which case no row was touched.""" 

481 user_ids: Final = frozenset(u.user_id for u in users) 

482 try: 

483 deletion: Final = await _delete_users_tx(prisma_client, users, teams_of, user_api_key_dict, litellm_changed_by) 

484 except Exception as e: # noqa: BLE001 # the rolled-back batch is reported per row, not as a request failure 

485 verbose_proxy_logger.error("users/bulk_delete: failed to delete users %s: %s", sorted(user_ids), e) 

486 return _error_message(e) 

487 if deletion.deleted_keys: 

488 KeyManagementEventHooks.create_key_deleted_audit_logs( 

489 keys_being_deleted=deletion.deleted_keys, 

490 user_api_key_dict=user_api_key_dict, 

491 litellm_changed_by=litellm_changed_by, 

492 ) 

493 await delete_cache_key_objects( 

494 hashed_tokens=deletion.deleted_key_tokens, 

495 user_api_key_cache=user_api_key_cache, 

496 proxy_logging_obj=proxy_logging_obj, 

497 ) 

498 await evict_and_broadcast(cache_keys=deletion.jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache) 

499 await evict_and_broadcast(cache_keys=sorted(user_ids), user_api_key_cache=user_api_key_cache) 

500 for removal in deletion.removals.values(): 

501 _emit_team_members_metric(removal.team) 

502 audit_outcomes: Final = await _bounded( 

503 UserManagementEventHooks.create_internal_user_audit_log( 

504 user_id=u.user_id, 

505 action="deleted", 

506 litellm_changed_by=litellm_changed_by, 

507 user_api_key_dict=user_api_key_dict, 

508 litellm_proxy_admin_name=litellm_proxy_admin_name, 

509 before_value=u.model_dump_json(exclude_none=True), 

510 ) 

511 for u in users 

512 ) 

513 for u, outcome in zip(users, audit_outcomes, strict=True): 

514 if isinstance(outcome, BaseException): 

515 verbose_proxy_logger.warning("Failed to create audit log for user %s: %s", u.user_id, outcome) 

516 return deletion 

517 

518 

519async def bulk_delete_users( 

520 data: BulkDeleteUserRequest, 

521 user_api_key_dict: UserAPIKeyAuth, 

522 prisma_client: PrismaClient, 

523 user_api_key_cache: UserApiKeyCache, 

524 proxy_logging_obj: ProxyLogging | None, 

525 litellm_proxy_admin_name: str | None, 

526 litellm_changed_by: str | None, 

527) -> tuple[UserDeleteResult, ...]: 

528 caller_is_proxy_admin: Final = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value 

529 caller_admin_org_ids: Final = await _caller_admin_org_ids(prisma_client, user_api_key_dict) 

530 if not caller_is_proxy_admin and not caller_admin_org_ids: 530 ↛ 531line 530 didn't jump to line 531 because the condition on line 530 was never true

531 raise _forbidden("Only PROXY_ADMIN or ORG_ADMIN users may delete users.") 

532 

533 unique_ids: Final = frozenset(data.user_ids) 

534 rows: Final = await UserRepository(prisma_client).table.find_many(where=_in_filter("user_id", unique_ids)) 

535 rows_by_id: Final = MappingProxyType({row.user_id: row for row in rows}) 

536 target_memberships: Final = ( 

537 () 

538 if caller_is_proxy_admin 

539 else await OrganizationMembershipRepository(prisma_client).table.find_many( 

540 where=_in_filter("user_id", unique_ids) 

541 ) 

542 ) 

543 

544 def precheck_error(user_id: str) -> str | None: 

545 if user_id not in rows_by_id: 545 ↛ 547line 545 didn't jump to line 547 because the condition on line 545 was always true

546 return f"User id={user_id} not found" 

547 if caller_is_proxy_admin: 

548 return None 

549 org_ids: Final = frozenset( 

550 m.organization_id for m in target_memberships if m.user_id == user_id and m.organization_id 

551 ) 

552 return _scope_error(user_id, org_ids, caller_admin_org_ids) 

553 

554 precheck_errors: Final = MappingProxyType({uid: precheck_error(uid) for uid in unique_ids}) 

555 candidates: Final = tuple(rows_by_id[uid] for uid in sorted(unique_ids) if precheck_errors[uid] is None) 

556 candidate_ids: Final = frozenset(u.user_id for u in candidates) 

557 

558 memberships: Final = await TeamMembershipRepository(prisma_client).table.find_many( 

559 where=_in_filter("user_id", candidate_ids) 

560 ) 

561 teams_of: Final = MappingProxyType( 

562 { 

563 u.user_id: frozenset(u.teams) | frozenset(m.team_id for m in memberships if m.user_id == u.user_id) 

564 for u in candidates 

565 } 

566 ) 

567 deletion: Final = ( 

568 await _delete_users( 

569 prisma_client, 

570 candidates, 

571 teams_of, 

572 user_api_key_dict, 

573 user_api_key_cache, 

574 proxy_logging_obj, 

575 litellm_proxy_admin_name, 

576 litellm_changed_by, 

577 ) 

578 if candidates 

579 else _UserBatchDeletion(removals=MappingProxyType({}), deleted_keys=(), jwt_mapping_cache_keys=()) 

580 ) 

581 

582 def result(index: int, user_id: str) -> UserDeleteResult: 

583 if user_id in data.user_ids[:index]: 

584 return UserDeleteResult(user_id=user_id, success=False, error=f"Duplicate user_id in request: {user_id}") 

585 error: Final = precheck_errors[user_id] 

586 if error is not None: 586 ↛ 588line 586 didn't jump to line 588 because the condition on line 586 was always true

587 return UserDeleteResult(user_id=user_id, success=False, error=error) 

588 if isinstance(deletion, str): 

589 return UserDeleteResult( 

590 user_id=user_id, 

591 user_email=rows_by_id[user_id].user_email, 

592 success=False, 

593 error=f"Failed to delete user: {deletion}", 

594 ) 

595 return UserDeleteResult( 

596 user_id=user_id, 

597 user_email=rows_by_id[user_id].user_email, 

598 success=True, 

599 teams_removed=tuple(tid for tid, r in deletion.removals.items() if user_id in r.removed), 

600 ) 

601 

602 return tuple(result(i, uid) for i, uid in enumerate(data.user_ids))