Coverage for .venv/lib/python3.13/site-packages/litellm/proxy/analytics_endpoints/cache_activity.py: 82%

1import asyncio 

2import json 

3from collections.abc import Sequence 

4from datetime import datetime 

5from typing import Final, Protocol 

6 

7from pydantic import BaseModel, TypeAdapter 

8 

9from litellm.proxy._types import LiteLLMRoutes 

10 

11UNKNOWN_CALL_TYPE: Final = "Unknown" 

12INFO_ROUTES_JSON: Final = json.dumps(LiteLLMRoutes.info_routes.value) 

13 

14 

15class _SupportsQueryRaw(Protocol): 

16 """The single database operation the cache-activity queries issue.""" 

17 

18 async def query_raw(self, query: str, *args: object) -> Sequence[object]: ... 18 ↛ exitline 18 didn't return from function 'query_raw' because

19 

20 

21class _SupportsRawQueryDb(Protocol): 

22 """A prisma client handle, narrowed to the raw-query surface used here.""" 

23 

24 @property 

25 def db(self) -> _SupportsQueryRaw: ... 25 ↛ exitline 25 didn't return from function 'db' because

26 

27 

28class CacheActivityGroup(BaseModel): 

29 call_type: str 

30 api_requests: int 

31 cache_hits: int 

32 failed_requests: int 

33 cached_completion_tokens: int 

34 generated_completion_tokens: int 

35 

36 

37class CacheActivityTotals(BaseModel): 

38 api_requests: int 

39 cache_hits: int 

40 failed_requests: int 

41 cached_completion_tokens: int 

42 cache_hit_ratio: float 

43 

44 

45class CacheActivityFilterOptions(BaseModel): 

46 key_aliases: list[str] 

47 models: list[str] 

48 

49 

50class CacheActivityErrorBucket(BaseModel): 

51 call_type: str 

52 error_code: str 

53 error_class: str 

54 count: int 

55 

56 

57class CacheActivityResponse(BaseModel): 

58 groups: list[CacheActivityGroup] 

59 totals: CacheActivityTotals 

60 filter_options: CacheActivityFilterOptions 

61 error_breakdown: tuple[CacheActivityErrorBucket, ...] 

62 

63 

64GROUPS_SQL: Final = """ 

65 SELECT 

66 CASE WHEN sl."call_type" = '' THEN 'Unknown' ELSE sl."call_type" END AS call_type, 

67 (COUNT(*) 

68 - SUM(CASE WHEN COALESCE(sl."cache_hit", '') = 'True' THEN 1 ELSE 0 END) 

69 - SUM(CASE WHEN sl."status" = 'failure' THEN 1 ELSE 0 END))::int AS api_requests, 

70 SUM(CASE WHEN COALESCE(sl."cache_hit", '') = 'True' THEN 1 ELSE 0 END)::int AS cache_hits, 

71 SUM(CASE WHEN sl."status" = 'failure' THEN 1 ELSE 0 END)::int AS failed_requests, 

72 SUM(CASE WHEN COALESCE(sl."cache_hit", '') = 'True' THEN sl."completion_tokens" ELSE 0 END)::int 

73 AS cached_completion_tokens, 

74 SUM(CASE WHEN COALESCE(sl."cache_hit", '') != 'True' THEN sl."completion_tokens" ELSE 0 END)::int 

75 AS generated_completion_tokens 

76 FROM "LiteLLM_SpendLogs" sl 

77 LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token" 

78 WHERE 

79 sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') 

80 AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') 

81 AND ($3::jsonb = '[]'::jsonb 

82 OR COALESCE(vt."key_alias", 'Unnamed Key') IN (SELECT jsonb_array_elements_text($3::jsonb))) 

83 AND ($4::jsonb = '[]'::jsonb 

84 OR sl."model" IN (SELECT jsonb_array_elements_text($4::jsonb))) 

85 AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($5::jsonb)) 

86 GROUP BY 1 

87 ORDER BY (COUNT(*)) DESC 

88""" 

89 

90ERROR_BREAKDOWN_SQL: Final = """ 

91 SELECT 

92 CASE WHEN sl."call_type" = '' THEN 'Unknown' ELSE sl."call_type" END AS call_type, 

93 COALESCE(NULLIF(sl."metadata"->'error_information'->>'error_code', ''), 'Unknown') AS error_code, 

94 COALESCE(NULLIF(sl."metadata"->'error_information'->>'error_class', ''), 'Unknown') AS error_class, 

95 COUNT(*)::int AS count 

96 FROM "LiteLLM_SpendLogs" sl 

97 LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token" 

98 WHERE 

99 sl."status" = 'failure' 

100 AND sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') 

101 AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') 

102 AND ($3::jsonb = '[]'::jsonb 

103 OR COALESCE(vt."key_alias", 'Unnamed Key') IN (SELECT jsonb_array_elements_text($3::jsonb))) 

104 AND ($4::jsonb = '[]'::jsonb 

105 OR sl."model" IN (SELECT jsonb_array_elements_text($4::jsonb))) 

106 AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($5::jsonb)) 

107 GROUP BY 1, 2, 3 

108 ORDER BY (COUNT(*)) DESC 

109""" 

110 

111KEY_ALIAS_OPTIONS_SQL: Final = """ 

112 SELECT DISTINCT COALESCE(vt."key_alias", 'Unnamed Key') AS key_alias 

113 FROM "LiteLLM_SpendLogs" sl 

114 LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token" 

115 WHERE 

116 sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') 

117 AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') 

118 AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($3::jsonb)) 

119 ORDER BY 1 

120""" 

121 

122MODEL_OPTIONS_SQL: Final = """ 

123 SELECT DISTINCT sl."model" AS model 

124 FROM "LiteLLM_SpendLogs" sl 

125 WHERE 

126 sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC') 

127 AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') 

128 AND sl."model" != '' 

129 AND sl."call_type" NOT IN (SELECT jsonb_array_elements_text($3::jsonb)) 

130 ORDER BY 1 

131""" 

132 

133 

134class _KeyAliasRow(BaseModel): 

135 key_alias: str 

136 

137 

138class _ModelRow(BaseModel): 

139 model: str 

140 

141 

142_groups_adapter: Final = TypeAdapter(list[CacheActivityGroup]) 

143_error_buckets_adapter: Final = TypeAdapter(tuple[CacheActivityErrorBucket, ...]) 

144_key_alias_rows_adapter: Final = TypeAdapter(list[_KeyAliasRow]) 

145_model_rows_adapter: Final = TypeAdapter(list[_ModelRow]) 

146 

147 

148def compute_totals(groups: Sequence[CacheActivityGroup]) -> CacheActivityTotals: 

149 api_requests: Final = sum(group.api_requests for group in groups) 

150 cache_hits: Final = sum(group.cache_hits for group in groups) 

151 failed_requests: Final = sum(group.failed_requests for group in groups) 

152 all_requests: Final = api_requests + cache_hits + failed_requests 

153 return CacheActivityTotals( 

154 api_requests=api_requests, 

155 cache_hits=cache_hits, 

156 failed_requests=failed_requests, 

157 cached_completion_tokens=sum(group.cached_completion_tokens for group in groups), 

158 cache_hit_ratio=(cache_hits / all_requests) * 100 if all_requests > 0 else 0.0, 

159 ) 

160 

161 

162async def get_cache_activity( 

163 prisma_client: _SupportsRawQueryDb, 

164 start_date: datetime, 

165 end_date: datetime, 

166 key_aliases: Sequence[str], 

167 models: Sequence[str], 

168) -> CacheActivityResponse: 

169 key_aliases_json: Final = json.dumps(list(key_aliases)) 

170 models_json: Final = json.dumps(list(models)) 

171 group_rows, error_rows, key_alias_rows, model_rows = await asyncio.gather( 

172 prisma_client.db.query_raw(GROUPS_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON), 

173 prisma_client.db.query_raw( 

174 ERROR_BREAKDOWN_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON 

175 ), 

176 prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON), 

177 prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON), 

178 ) 

179 groups: Final = _groups_adapter.validate_python(group_rows or []) 

180 return CacheActivityResponse( 

181 groups=groups, 

182 totals=compute_totals(groups), 

183 filter_options=CacheActivityFilterOptions( 

184 key_aliases=[row.key_alias for row in _key_alias_rows_adapter.validate_python(key_alias_rows or [])], 

185 models=[row.model for row in _model_rows_adapter.validate_python(model_rows or [])], 

186 ), 

187 error_breakdown=_error_buckets_adapter.validate_python(error_rows or []), 

188 )