Coverage for .venv/lib/python3.13/site-packages/litellm/proxy/caching_routes.py: 61%

1from typing import Final 

2 

3from fastapi import APIRouter, Depends, HTTPException, Request 

4 

5import litellm 

6from litellm._logging import verbose_proxy_logger 

7from litellm.caching.caching import RedisCache 

8from litellm.litellm_core_utils.safe_json_dumps import safe_dumps 

9from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker 

10from litellm.proxy._types import ProxyErrorTypes, ProxyException 

11from litellm.proxy.auth.user_api_key_auth import user_api_key_auth 

12from litellm.types.caching import CachePingResponse, HealthCheckCacheParams 

13 

14masker: Final = SensitiveDataMasker() 

15 

16router: Final = APIRouter( 

17 prefix="/cache", 

18 tags=["caching"], 

19) 

20 

21 

22def _extract_cache_params() -> dict[str, object]: 

23 """ 

24 Safely extracts and cleans cache parameters. 

25 

26 The health check UI needs to display specific cache parameters, to show users how they set up their cache. 

27 

28 eg. 

29 { 

30 "host": "localhost", 

31 "port": 6379, 

32 "redis_kwargs": {"db": 0}, 

33 "namespace": "test", 

34 } 

35 

36 Returns: 

37 Dict containing cleaned and masked cache parameters 

38 """ 

39 if litellm.cache is None: 39 ↛ 40line 39 didn't jump to line 40 because the condition on line 39 was never true

40 return {} 

41 try: 

42 cache_params: Final = vars(litellm.cache.cache) 

43 cleaned_params: Final = HealthCheckCacheParams(**cache_params).model_dump() if cache_params else {} 

44 return masker.mask_dict(cleaned_params) 

45 except (AttributeError, TypeError) as e: 

46 verbose_proxy_logger.debug("Error extracting cache params: %s", e) 

47 return {} 

48 

49 

50@router.get( 

51 "/ping", 

52 response_model=CachePingResponse, 

53 dependencies=[Depends(user_api_key_auth)], 

54) 

55async def cache_ping(): 

56 """ 

57 Endpoint for checking if cache can be pinged 

58 """ 

59 litellm_cache_params: dict[str, object] = {} 

60 cleaned_cache_params: dict[str, object] = {} 

61 if litellm.cache is None: 61 ↛ 62line 61 didn't jump to line 62 because the condition on line 61 was never true

62 raise ProxyException( 

63 message=safe_dumps( 

64 { 

65 "message": "Cache not initialized. litellm.cache is None", 

66 "litellm_cache_params": "{}", 

67 "health_check_cache_params": "{}", 

68 } 

69 ), 

70 type=ProxyErrorTypes.cache_ping_error, 

71 param="cache_ping", 

72 code=503, 

73 ) 

74 try: 

75 litellm_cache_params = masker.mask_dict(vars(litellm.cache)) 

76 # remove field that might reference itself 

77 litellm_cache_params.pop("cache", None) 

78 cleaned_cache_params = _extract_cache_params() 

79 

80 if litellm.cache.type == "redis": 80 ↛ 81line 80 didn't jump to line 81 because the condition on line 80 was never true

81 ping_response: Final = await litellm.cache.ping() 

82 verbose_proxy_logger.debug("/cache/ping: ping_response: " + str(ping_response)) 

83 # add cache does not return anything 

84 await litellm.cache.async_add_cache( 

85 result="test_key", 

86 model="test-model", 

87 messages=[{"role": "user", "content": "test from litellm"}], 

88 ) 

89 verbose_proxy_logger.debug("/cache/ping: done with set_cache()") 

90 

91 return CachePingResponse( 

92 status="healthy", 

93 cache_type=str(litellm.cache.type), 

94 ping_response=True, 

95 set_cache_response="success", 

96 litellm_cache_params=safe_dumps(litellm_cache_params), 

97 health_check_cache_params=cleaned_cache_params, 

98 ) 

99 else: 

100 return CachePingResponse( 

101 status="healthy", 

102 cache_type=str(litellm.cache.type), 

103 litellm_cache_params=safe_dumps(litellm_cache_params), 

104 ) 

105 except HTTPException: 

106 raise 

107 except Exception: 

108 verbose_proxy_logger.exception("Cache health check failed") 

109 error_message: Final = { 

110 "message": "Service Unhealthy", 

111 "litellm_cache_params": safe_dumps(litellm_cache_params), 

112 "health_check_cache_params": safe_dumps(cleaned_cache_params), 

113 } 

114 raise ProxyException( 

115 message=safe_dumps(error_message), 

116 type=ProxyErrorTypes.cache_ping_error, 

117 param="cache_ping", 

118 code=503, 

119 ) 

120 

121 

122@router.post( 

123 "/delete", 

124 tags=["caching"], 

125 dependencies=[Depends(user_api_key_auth)], 

126) 

127async def cache_delete(request: Request): 

128 """ 

129 Endpoint for deleting a key from the cache. All responses from litellm proxy have `x-litellm-cache-key` in the headers 

130 

131 Parameters: 

132 - **keys**: *Optional[List[str]]* - A list of keys to delete from the cache. Example {"keys": ["key1", "key2"]} 

133 

134 ```shell 

135 curl -X POST "http://0.0.0.0:4000/cache/delete" \ 

136 -H "Authorization: Bearer sk-1234" \ 

137 -d '{"keys": ["key1", "key2"]}' 

138 ``` 

139 

140 """ 

141 try: 

142 if litellm.cache is None: 

143 raise HTTPException(status_code=503, detail="Cache not initialized. litellm.cache is None") 

144 

145 request_data: Final = await request.json() 

146 keys: Final = request_data.get("keys", None) 

147 

148 if litellm.cache.type == "redis": 

149 await litellm.cache.delete_cache_keys(keys=keys) 

150 return { 

151 "status": "success", 

152 } 

153 else: 

154 raise HTTPException( 

155 status_code=500, 

156 detail=f"Cache type {litellm.cache.type} does not support deleting a key. only `redis` is supported", 

157 ) 

158 except Exception as e: 

159 raise HTTPException( 

160 status_code=500, 

161 detail=f"Cache Delete Failed({e})", 

162 ) 

163 

164 

165def _get_redis_client_info(cache_instance: RedisCache) -> tuple[list[object], int]: 

166 """ 

167 Helper function to safely get Redis client list information. 

168 

169 Returns: 

170 tuple: (client_list, num_clients) where num_clients is -1 if CLIENT LIST is unavailable 

171 """ 

172 try: 

173 client_list: Final = cache_instance.client_list() 

174 return client_list, len(client_list) 

175 except Exception as e: 

176 verbose_proxy_logger.warning("CLIENT LIST command failed (likely restricted on managed Redis): %s", e) 

177 return ["CLIENT LIST command not available on this Redis instance"], -1 

178 

179 

180@router.get( 

181 "/redis/info", 

182 dependencies=[Depends(user_api_key_auth)], 

183) 

184async def cache_redis_info(): 

185 """ 

186 Endpoint for getting /redis/info 

187 """ 

188 try: 

189 if litellm.cache is None: 189 ↛ 190line 189 didn't jump to line 190 because the condition on line 189 was never true

190 raise HTTPException(status_code=503, detail="Cache not initialized. litellm.cache is None") 

191 

192 if not (litellm.cache.type == "redis" and isinstance(litellm.cache.cache, RedisCache)): 192 ↛ 199line 192 didn't jump to line 199 because the condition on line 192 was always true

193 raise HTTPException( 

194 status_code=500, 

195 detail=f"Cache type {litellm.cache.type} does not support redis info", 

196 ) 

197 

198 # Get client information (handles CLIENT LIST restrictions gracefully) 

199 client_list, num_clients = _get_redis_client_info(litellm.cache.cache) 

200 

201 # Get Redis server information 

202 redis_info: Final = litellm.cache.cache.info() 

203 

204 return { 

205 "num_clients": num_clients, 

206 "clients": client_list, 

207 "info": redis_info, 

208 } 

209 except Exception as e: 

210 raise HTTPException( 

211 status_code=503, 

212 detail=f"Service Unhealthy ({e})", 

213 ) 

214 

215 

216@router.post( 

217 "/flushall", 

218 tags=["caching"], 

219 dependencies=[Depends(user_api_key_auth)], 

220) 

221async def cache_flushall(): 

222 """ 

223 A function to flush all items from the cache. (All items will be deleted from the cache with this) 

224 Raises HTTPException if the cache is not initialized or if the cache type does not support flushing. 

225 Returns a dictionary with the status of the operation. 

226 

227 Usage: 

228 ``` 

229 curl -X POST http://0.0.0.0:4000/cache/flushall -H "Authorization: Bearer sk-1234" 

230 ``` 

231 """ 

232 try: 

233 if litellm.cache is None: 

234 raise HTTPException(status_code=503, detail="Cache not initialized. litellm.cache is None") 

235 if litellm.cache.type == "redis" and isinstance(litellm.cache.cache, RedisCache): 235 ↛ 236line 235 didn't jump to line 236 because the condition on line 235 was never true

236 litellm.cache.cache.flushall() 

237 return { 

238 "status": "success", 

239 } 

240 else: 

241 raise HTTPException( 

242 status_code=500, 

243 detail=f"Cache type {litellm.cache.type} does not support flushing", 

244 ) 

245 except Exception as e: 

246 raise HTTPException( 

247 status_code=503, 

248 detail=f"Service Unhealthy ({e})", 

249 )