Coverage for .venv/lib/python3.13/site-packages/litellm/proxy/auth/budget_throttle.py: 23%

1""" 

2Throttle a key after it exceeds its own ``max_budget`` instead of blocking it. 

3 

4When a key opts in via ``throttle_on_budget_exceeded`` and a global 

5``budget_exceeded_throttle_percentage`` is configured, an over-budget key keeps 

6serving requests but at a reduced TPM/RPM (the configured percentage of its 

7configured limits). The decision (over budget + opted in) is made once during 

8auth; the scaling is recomputed from the key's original limits on every request 

9so it never compounds across requests. 

10""" 

11 

12import math 

13from typing import Final 

14 

15import litellm 

16from litellm.proxy._types import UserAPIKeyAuth 

17 

18 

19def budget_throttle_percentage() -> float | None: 

20 """ 

21 The global throttle percentage, or None when throttling is disabled / 

22 misconfigured (in which case an over-budget key is hard-blocked, the safe 

23 default). 

24 """ 

25 pct: Final = litellm.budget_exceeded_throttle_percentage 

26 if not isinstance(pct, (int, float)) or isinstance(pct, bool): 

27 return None 

28 if not 0 < pct <= 1: 

29 return None 

30 return float(pct) 

31 

32 

33def should_throttle_budget_exceeded(valid_token: UserAPIKeyAuth) -> bool: 

34 """ 

35 True when a key that exceeded its own ``max_budget`` should be throttled 

36 rather than blocked: it opted in, a valid global percentage is set, and the 

37 key has a TPM or RPM limit to scale down. A key with neither limit has 

38 nothing to throttle, so it stays hard-blocked (the safe default) rather than 

39 serving unlimited requests past its budget. 

40 """ 

41 if (valid_token.metadata or {}).get("throttle_on_budget_exceeded") is not True: 

42 return False 

43 if valid_token.tpm_limit is None and valid_token.rpm_limit is None: 

44 return False 

45 return budget_throttle_percentage() is not None 

46 

47 

48def throttled_limit(limit: int | None, pct: float | None) -> int | None: 

49 """ 

50 Scale a TPM/RPM limit to ``pct`` of its value, keeping a trickle of at least 

51 1 so a throttled key is slowed rather than fully locked out. An unset limit 

52 or unset percentage leaves the limit unchanged. 

53 """ 

54 if limit is None or pct is None: 

55 return limit 

56 return max(1, math.floor(limit * pct))