Coverage for .venv/lib/python3.13/site-packages/litellm/proxy/response_api_endpoints/endpoints.py: 42%

1import asyncio 

2import contextlib 

3import json 

4import time 

5from collections.abc import AsyncIterator, Awaitable, Mapping, Sequence 

6from enum import Enum 

7from functools import partial 

8from types import MappingProxyType 

9from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeAlias, cast, get_args 

10from uuid import uuid4 

11 

12import fastapi 

13from fastapi import APIRouter, Depends, HTTPException, Request, Response 

14from fastapi.responses import JSONResponse 

15from openai.types.responses import ResponseItemList 

16from openai.types.responses.response_create_params import ResponseInputParam 

17from pydantic import BaseModel, ConfigDict, ValidationError 

18from starlette.websockets import WebSocket, WebSocketDisconnect 

19from typing_extensions import ReadOnly, TypedDict 

20 

21from litellm._logging import verbose_proxy_logger 

22from litellm.constants import EMPTY_MAPPING 

23from litellm.integrations.custom_guardrail import ModifyResponseException 

24from litellm.llms.base_llm.guardrail_translation.utils import ( 

25 blocked_responses_api_usage as _blocked_responses_api_usage, 

26) 

27from litellm.proxy._types import * 

28from litellm.proxy.auth.user_api_key_auth import ( 

29 UserAPIKeyAuth, 

30 user_api_key_auth, 

31 user_api_key_auth_websocket, 

32) 

33from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing 

34from litellm.proxy.common_utils.http_parsing_utils import ( 

35 _read_request_body, 

36 _safe_set_request_parsed_body, 

37) 

38from litellm.proxy.route_llm_request import raise_if_required_body_param_missing 

39from litellm.types.llms.openai import ( 

40 REASONING_EFFORT, 

41 ResponsesAPIOptionalRequestParams, 

42 ResponsesAPIResponse, 

43) 

44from litellm.types.responses.main import DeleteResponseResult 

45from litellm.types.utils import TokenCountResponse 

46 

47if TYPE_CHECKING: 47 ↛ 48line 47 didn't jump to line 48 because the condition on line 47 was never true

48 from litellm.router import Router 

49 

50router: Final = APIRouter() 

51 

52_ResponseDocSchemas: TypeAlias = dict[int | str, dict[str, object]] # fastapi's responses kwarg 

53 

54RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {200: {"model": ResponsesAPIResponse}} 

55RESPONSES_API_CREATE_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = { 

56 200: { 

57 "model": ResponsesAPIResponse, 

58 "content": { 

59 "text/event-stream": {"schema": {"type": "string", "description": "Server sent events when stream=true"}} 

60 }, 

61 } 

62} 

63DELETE_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {200: {"model": DeleteResponseResult}} 

64RESPONSE_ITEM_LIST_SCHEMAS: Final[_ResponseDocSchemas] = {200: {"model": ResponseItemList}} 

65 

66_user_api_key_auth_dep: Final = Depends(user_api_key_auth) 

67_RESPONSES_TAGS: Final[list[str | Enum]] = ["responses"] # mutable-ok: fastapi's route signature requires list tags 

68 

69_TOOL_PAYLOAD_KEYS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( 

70 { 

71 "custom": ("name", "description", "format"), 

72 "function": ("name", "description", "parameters", "strict"), 

73 } 

74) 

75_EMPTY_TOOL_PAYLOAD: Final[Mapping[str, object]] = MappingProxyType({}) 

76 

77 

78def _convert_tool_payload_value(key: str, value: object, *, to_chat: bool) -> object: 

79 if key != "format" or not isinstance(value, dict): 

80 return value 

81 from litellm.litellm_core_utils.prompt_templates.common_utils import ( 

82 convert_custom_tool_format_to_chat_shape, 

83 convert_custom_tool_format_to_responses_shape, 

84 ) 

85 

86 convert = convert_custom_tool_format_to_chat_shape if to_chat else convert_custom_tool_format_to_responses_shape 

87 return convert(value) 

88 

89 

90def _convert_tool_envelope(obj: object, *, to_chat: bool) -> object: 

91 if not isinstance(obj, dict): 91 ↛ 93line 91 didn't jump to line 93 because the condition on line 91 was always true

92 return obj 

93 tool_type: Final = obj.get("type") 

94 payload_keys: Final = _TOOL_PAYLOAD_KEYS.get(tool_type) 

95 if payload_keys is None: 

96 return obj 

97 nested: Final = obj.get(tool_type) 

98 nested_source: Final = nested if isinstance(nested, dict) else _EMPTY_TOOL_PAYLOAD 

99 payload: Final = { # mutable-ok: tool entries are embedded verbatim in the JSON request body 

100 key: _convert_tool_payload_value(key, nested_source[key] if key in nested_source else obj[key], to_chat=to_chat) 

101 for key in payload_keys 

102 if key in nested_source or key in obj 

103 } 

104 if "name" not in payload: 

105 return obj 

106 return {"type": tool_type, tool_type: payload} if to_chat else {"type": tool_type, **payload} # mutable-ok: same 

107 

108 

109def _normalize_tool_dialect( 

110 data: dict, *, to_chat: bool 

111) -> dict: # mutable-ok: the parsed request body contract is a plain dict 

112 tools: Final = data.get("tools") 

113 tool_choice: Final = data.get("tool_choice") 

114 normalized_tools: Final = ( 

115 [_convert_tool_envelope(tool, to_chat=to_chat) for tool in tools] if isinstance(tools, list) else tools 

116 ) 

117 normalized_choice: Final = _convert_tool_envelope(tool_choice, to_chat=to_chat) 

118 if normalized_tools == tools and normalized_choice == tool_choice: 118 ↛ 120line 118 didn't jump to line 120 because the condition on line 118 was always true

119 return data 

120 replaceable: Final = (("tools", normalized_tools), ("tool_choice", normalized_choice)) 

121 return {**data, **{key: value for key, value in replaceable if key in data}} # mutable-ok: plain body dict 

122 

123 

124def _is_chat_completions_body(data: Mapping[str, object]) -> bool: 

125 messages: Final = data.get("messages") 

126 if isinstance(messages, list) and messages: 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true

127 return True 

128 return "messages" in data and "input" not in data 

129 

130 

131_CURSOR_THINKING_SEPARATOR: Final = "-thinking-" 

132_CURSOR_FAST_SUFFIX: Final = "-fast" 

133_CURSOR_THINKING_LEVELS: Final[frozenset[str]] = frozenset(get_args(REASONING_EFFORT)) 

134 

135 

136class _CursorModelVariant(NamedTuple): 

137 base_model: str 

138 reasoning_effort: str | None 

139 

140 

141def _parse_cursor_model_variant(model: str) -> _CursorModelVariant: 

142 stripped: Final = model.removesuffix(_CURSOR_FAST_SUFFIX) 

143 base, separator, level = stripped.rpartition(_CURSOR_THINKING_SEPARATOR) 

144 if separator and base and level in _CURSOR_THINKING_LEVELS: 

145 return _CursorModelVariant(base, level) 

146 return _CursorModelVariant(stripped, None) 

147 

148 

149def _router_can_serve(model: str, llm_router: "Router | None") -> bool: 

150 if llm_router is None: 

151 return False 

152 if llm_router.is_recognized_model(model): 

153 return True 

154 if model in llm_router.team_public_model_names: 

155 return True 

156 return bool(llm_router.pattern_router.get_pattern(model)) 

157 

158 

159def _resolve_cursor_model_variant( 

160 data: dict, llm_router: "Router | None" 

161) -> dict: # mutable-ok: the parsed request body contract is a plain dict 

162 model: Final = data.get("model") 

163 if not isinstance(model, str) or _router_can_serve(model, llm_router): 163 ↛ 165line 163 didn't jump to line 165 because the condition on line 163 was always true

164 return data 

165 variant: Final = _parse_cursor_model_variant(model) 

166 if variant.base_model == model or not _router_can_serve(variant.base_model, llm_router): 

167 return data 

168 resolved: Final = {**data, "model": variant.base_model} # mutable-ok: plain body dict 

169 if variant.reasoning_effort is None: 

170 return resolved 

171 if _is_chat_completions_body(data): 

172 if "reasoning_effort" in data: 

173 return resolved 

174 return {**resolved, "reasoning_effort": variant.reasoning_effort} # mutable-ok: plain body dict 

175 reasoning: Final = data.get("reasoning") 

176 if isinstance(reasoning, dict): 

177 if reasoning.get("effort"): 

178 return resolved 

179 return {**resolved, "reasoning": {**reasoning, "effort": variant.reasoning_effort}} # mutable-ok: same 

180 return {**resolved, "reasoning": {"effort": variant.reasoning_effort}} # mutable-ok: plain body dict 

181 

182 

183async def _resolve_cursor_model_variant_before_auth(request: Request) -> None: 

184 from litellm.proxy.proxy_server import llm_router 

185 

186 try: 

187 raw_body: Final = await _read_request_body(request=request) 

188 except (json.JSONDecodeError, ProxyException): 

189 return 

190 resolved: Final = _resolve_cursor_model_variant(raw_body, llm_router) 

191 if resolved is not raw_body: 191 ↛ 192line 191 didn't jump to line 192 because the condition on line 191 was never true

192 _safe_set_request_parsed_body(request=request, parsed_body=resolved) 

193 

194 

195@router.post( 

196 "/v1/responses", 

197 dependencies=[Depends(user_api_key_auth)], 

198 tags=["responses"], 

199 responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, 

200) 

201@router.post( 

202 "/responses", 

203 dependencies=[Depends(user_api_key_auth)], 

204 tags=["responses"], 

205 responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, 

206) 

207@router.post( 

208 "/openai/v1/responses", 

209 dependencies=[Depends(user_api_key_auth)], 

210 tags=["responses"], 

211 responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, 

212) 

213async def responses_api( 

214 request: Request, 

215 fastapi_response: Response, 

216 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

217): 

218 """ 

219 Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses 

220 

221 Supports background mode with polling_via_cache for partial response retrieval. 

222 When background=true and polling_via_cache is enabled, returns a polling_id immediately 

223 and streams the response in the background, updating Redis cache. 

224 

225 ```bash 

226 # Normal request 

227 curl -X POST http://localhost:4000/v1/responses \ 

228 -H "Content-Type: application/json" \ 

229 -H "Authorization: Bearer sk-1234" \ 

230 -d '{ 

231 "model": "gpt-4o", 

232 "input": "Tell me about AI" 

233 }' 

234 

235 # Background request with polling 

236 curl -X POST http://localhost:4000/v1/responses \ 

237 -H "Content-Type: application/json" \ 

238 -H "Authorization: Bearer sk-1234" \ 

239 -d '{ 

240 "model": "gpt-4o", 

241 "input": "Tell me about AI", 

242 "background": true 

243 }' 

244 ``` 

245 """ 

246 from litellm.proxy.proxy_server import ( 

247 _read_request_body, 

248 general_settings, 

249 llm_router, 

250 native_background_mode, 

251 polling_cache_ttl, 

252 polling_via_cache_enabled, 

253 proxy_config, 

254 proxy_logging_obj, 

255 redis_usage_cache, 

256 select_data_generator, 

257 user_api_base, 

258 user_max_tokens, 

259 user_model, 

260 user_request_timeout, 

261 user_temperature, 

262 version, 

263 ) 

264 

265 native_data_generator: Final = partial(select_data_generator, responses_stream_errors=True) 

266 data = await _read_request_body(request=request) 

267 

268 # Check if polling via cache should be used for this request 

269 from litellm.proxy.response_polling.polling_handler import ( 

270 should_use_polling_for_request, 

271 ) 

272 

273 should_use_polling: Final = should_use_polling_for_request( 

274 background_mode=data.get("background", False), 

275 polling_via_cache_enabled=polling_via_cache_enabled, 

276 redis_cache=redis_usage_cache, 

277 model=data.get("model", ""), 

278 llm_router=llm_router, 

279 native_background_mode=native_background_mode, 

280 ) 

281 

282 # If polling is enabled, use polling mode 

283 if should_use_polling: 283 ↛ 284line 283 didn't jump to line 284 because the condition on line 283 was never true

284 from litellm.proxy.response_polling.background_streaming import ( 

285 background_streaming_task, 

286 ) 

287 from litellm.proxy.response_polling.polling_handler import ( 

288 ResponsePollingHandler, 

289 ) 

290 

291 verbose_proxy_logger.info("Starting background response with polling for model=%s", data.get("model")) 

292 

293 # Run pre-call checks (rate limits, guardrails, budget) BEFORE creating 

294 # polling ID. This ensures rate-limited requests get a synchronous 429 

295 # instead of a polling ID that immediately fails in the background task. 

296 processor = ProxyBaseLLMRequestProcessing(data=data) 

297 try: 

298 data, _logging_obj = await processor.common_processing_pre_call_logic( 

299 request=request, 

300 general_settings=general_settings, 

301 proxy_logging_obj=proxy_logging_obj, 

302 user_api_key_dict=user_api_key_dict, 

303 version=version, 

304 proxy_config=proxy_config, 

305 user_model=user_model, 

306 user_temperature=user_temperature, 

307 user_request_timeout=user_request_timeout, 

308 user_max_tokens=user_max_tokens, 

309 user_api_base=user_api_base, 

310 model=None, 

311 route_type="aresponses", 

312 llm_router=llm_router, 

313 ) 

314 raise_if_required_body_param_missing(route_type="aresponses", data=data) 

315 except Exception as e: 

316 raise await processor._handle_llm_api_exception( 

317 e=e, 

318 user_api_key_dict=user_api_key_dict, 

319 proxy_logging_obj=proxy_logging_obj, 

320 version=version, 

321 ) 

322 

323 # Initialize polling handler with configured TTL (from global config) 

324 polling_handler: Final = ResponsePollingHandler( 

325 redis_cache=redis_usage_cache, 

326 ttl=polling_cache_ttl, # Global var set at startup 

327 ) 

328 

329 # Generate polling ID 

330 polling_id: Final = ResponsePollingHandler.generate_polling_id() 

331 

332 # Create initial state in Redis 

333 initial_state: Final = await polling_handler.create_initial_state( 

334 polling_id=polling_id, 

335 request_data=data, 

336 ) 

337 

338 # Start background task to stream and update cache. 

339 # Pass pre-processed data so the background task skips pre-call logic 

340 # (rate limits, guardrails already checked above). 

341 asyncio.create_task( 

342 background_streaming_task( 

343 polling_id=polling_id, 

344 data=data.copy(), 

345 polling_handler=polling_handler, 

346 request=request, 

347 fastapi_response=fastapi_response, 

348 user_api_key_dict=user_api_key_dict, 

349 general_settings=general_settings, 

350 llm_router=llm_router, 

351 proxy_config=proxy_config, 

352 proxy_logging_obj=proxy_logging_obj, 

353 select_data_generator=native_data_generator, 

354 user_model=user_model, 

355 user_temperature=user_temperature, 

356 user_request_timeout=user_request_timeout, 

357 user_max_tokens=user_max_tokens, 

358 user_api_base=user_api_base, 

359 version=version, 

360 ) 

361 ) 

362 

363 # Return OpenAI Response object format (initial state) 

364 # https://platform.openai.com/docs/api-reference/responses/object 

365 return initial_state 

366 

367 # Normal response flow 

368 processor = ProxyBaseLLMRequestProcessing(data=data) 

369 try: 

370 response: Final = await processor.base_process_llm_request( 

371 request=request, 

372 fastapi_response=fastapi_response, 

373 user_api_key_dict=user_api_key_dict, 

374 route_type="aresponses", 

375 proxy_logging_obj=proxy_logging_obj, 

376 llm_router=llm_router, 

377 general_settings=general_settings, 

378 proxy_config=proxy_config, 

379 select_data_generator=native_data_generator, 

380 model=None, 

381 user_model=user_model, 

382 user_temperature=user_temperature, 

383 user_request_timeout=user_request_timeout, 

384 user_max_tokens=user_max_tokens, 

385 user_api_base=user_api_base, 

386 version=version, 

387 ) 

388 

389 # Store in managed objects table if background mode is enabled 

390 if data.get("background") and isinstance(response, ResponsesAPIResponse): 

391 if response.status in ["queued", "in_progress"]: 

392 from litellm_enterprise.proxy.hooks.managed_files import ( 

393 _PROXY_LiteLLMManagedFiles, 

394 ) 

395 

396 managed_files_obj: Final = cast( 

397 _PROXY_LiteLLMManagedFiles | None, 

398 proxy_logging_obj.get_proxy_hook("managed_files"), 

399 ) 

400 

401 if managed_files_obj and llm_router: 

402 try: 

403 # Get the actual deployment model_id from hidden params 

404 hidden_params: Final = getattr(response, "_hidden_params", {}) or {} 

405 model_id: Final = hidden_params.get("model_id", None) 

406 

407 if not model_id: 

408 verbose_proxy_logger.warning( 

409 "No model_id found in response hidden params for response %s, skipping managed object storage", 

410 response.id, 

411 ) 

412 raise Exception("No model_id found in response hidden params") 

413 # Store in managed objects table 

414 await managed_files_obj.store_unified_object_id( 

415 unified_object_id=response.id, 

416 file_object=response, 

417 litellm_parent_otel_span=None, 

418 model_object_id=response.id, 

419 file_purpose="response", 

420 user_api_key_dict=user_api_key_dict, 

421 ) 

422 

423 verbose_proxy_logger.info( 

424 "Stored background response %s in managed objects table with unified_id=%s", 

425 response.id, 

426 response.id, 

427 ) 

428 except Exception as e: 

429 verbose_proxy_logger.error( 

430 "Failed to store background response in managed objects table: %s", e 

431 ) 

432 

433 return response 

434 except ModifyResponseException as e: 

435 # Guardrail passthrough: return violation message in Responses API format (200) 

436 _data: Final = e.request_data 

437 await proxy_logging_obj.post_call_failure_hook( 

438 user_api_key_dict=user_api_key_dict, 

439 original_exception=e, 

440 request_data=_data, 

441 ) 

442 

443 violation_text: Final = e.message 

444 response_obj: Final = ResponsesAPIResponse( 

445 id=f"resp_{uuid4()}", 

446 object="response", 

447 created_at=int(time.time()), 

448 model=e.model or data.get("model"), 

449 output=cast(Any, [{"content": [{"type": "text", "text": violation_text}]}]), 

450 status="completed", 

451 usage=_blocked_responses_api_usage(e.original_response), 

452 ) 

453 return response_obj 

454 except Exception as e: 

455 raise await processor._handle_llm_api_exception( 

456 e=e, 

457 user_api_key_dict=user_api_key_dict, 

458 proxy_logging_obj=proxy_logging_obj, 

459 version=version, 

460 ) 

461 

462 

463@router.get( 

464 "/cursor/models", 

465 dependencies=(_user_api_key_auth_dep,), 

466 tags=_RESPONSES_TAGS, 

467) 

468@router.get( 

469 "/cursor/v1/models", 

470 dependencies=(_user_api_key_auth_dep,), 

471 tags=_RESPONSES_TAGS, 

472) 

473async def cursor_model_list( 

474 user_api_key_dict: UserAPIKeyAuth = _user_api_key_auth_dep, 

475): 

476 """ 

477 OpenAI-compatible model listing for the Cursor BYOK base URL. 

478 

479 Clients pointed at `<proxy>/cursor` as an OpenAI-compatible base URL resolve and 

480 verify models via `GET {base}/models` (the OpenAI SDK contract). Without this 

481 route those requests fall through to the Cursor Cloud Agents passthrough, which 

482 demands a Cursor API key and 401s, so key verification silently fails before any 

483 chat request is ever sent. Delegates to the standard `/v1/models` handler. 

484 """ 

485 from litellm.proxy.proxy_server import model_list 

486 

487 return await model_list(user_api_key_dict=user_api_key_dict) 

488 

489 

490@router.post( 

491 "/cursor/chat/completions", 

492 dependencies=[ 

493 Depends(_resolve_cursor_model_variant_before_auth), 

494 Depends(user_api_key_auth), 

495 ], 

496 tags=["responses"], 

497) 

498async def cursor_chat_completions( 

499 request: Request, 

500 fastapi_response: Response, 

501 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

502): 

503 """ 

504 Cursor BYOK endpoint. Accepts both request shapes Cursor sends to its OpenAI-compatible 

505 base URL and always answers in chat completions format. 

506 

507 Cursor agent mode sends Responses API format bodies (`input`, flat tool defs, `reasoning`, 

508 custom tools) to the chat/completions path while expecting chat completions responses; 

509 those are routed through the Responses API pipeline and converted back. Genuine chat 

510 completions bodies (`messages` present) are routed through the standard chat completions 

511 pipeline, after normalizing each level of the `tools` array and `tool_choice` to the chat 

512 completions shapes OpenAI requires. Cursor mixes Responses API shapes into chat bodies 

513 per level, independently: a flat tool def (`{"type": "custom", "name": "ApplyPatch", ...}`) 

514 gets nested under `custom`, and a flat grammar format 

515 (`{"type": "grammar", "definition", "syntax"}`) gets wrapped as 

516 `{"type": "grammar", "grammar": {...}}` wherever it appears, including inside tool defs 

517 Cursor already sent pre-nested. 

518 

519 ```bash 

520 curl -X POST http://localhost:4000/cursor/chat/completions \ 

521 -H "Content-Type: application/json" \ 

522 -H "Authorization: Bearer sk-1234" \ 

523 -d '{ 

524 "model": "gpt-4o", 

525 "input": [{"role": "user", "content": "Hello"}] 

526 }' 

527 Responds back in chat completions format. 

528 ``` 

529 """ 

530 from litellm.completion_extras.litellm_responses_transformation.handler import ( 

531 responses_api_bridge, 

532 ) 

533 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper 

534 from litellm.proxy.proxy_server import ( 

535 async_data_generator, 

536 chat_completion, 

537 general_settings, 

538 llm_router, 

539 proxy_config, 

540 proxy_logging_obj, 

541 user_api_base, 

542 user_max_tokens, 

543 user_model, 

544 user_request_timeout, 

545 user_temperature, 

546 version, 

547 ) 

548 from litellm.types.llms.openai import ResponsesAPIResponse 

549 from litellm.types.utils import ModelResponse 

550 

551 raw_body: Final = await _read_request_body(request=request) 

552 

553 if _is_chat_completions_body(raw_body): 553 ↛ 558line 553 didn't jump to line 558 because the condition on line 553 was never true

554 # Genuine chat completions body (Cursor sends these for models whose BYOK it 

555 # already fixed); delegate so behavior matches /chat/completions exactly. 

556 # Keyed on messages CONTENT, not key presence: Cursor can send a null or 

557 # empty messages stub alongside a real agent-mode input array 

558 normalized: Final = _normalize_tool_dialect(raw_body, to_chat=True) 

559 if normalized is not raw_body: 

560 _safe_set_request_parsed_body(request=request, parsed_body=normalized) 

561 return await chat_completion( 

562 request=request, 

563 fastapi_response=fastapi_response, 

564 model=None, 

565 user_api_key_dict=user_api_key_dict, 

566 ) 

567 

568 # OpenAI's Responses API rejects chat-completions-only stream_options 

569 # (Cursor sends include_usage); usage arrives via response.completed anyway. 

570 # Rebuild rather than pop: _read_request_body can return the request-scope 

571 # cached parsed-body dict itself, and removing keys from it corrupts the 

572 # cache's key snapshot so later readers get an empty body 

573 body_without_stream_options: Final = { # mutable-ok: base_process_llm_request mutates the body dict in place 

574 key: value for key, value in raw_body.items() if key != "stream_options" 

575 } 

576 

577 data: Final = _normalize_tool_dialect(body_without_stream_options, to_chat=False) 

578 

579 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

580 

581 def cursor_data_generator(response, user_api_key_dict, request_data, request=None): 

582 """ 

583 Custom generator that transforms Responses API streaming chunks to chat completion chunks. 

584 

585 This generator is used for the cursor endpoint to convert Responses API format responses 

586 to chat completion format that Cursor IDE expects. 

587 

588 Args: 

589 response: The streaming Responses API event iterator (router-wrapped or not) 

590 user_api_key_dict: User API key authentication dict 

591 request_data: Request data containing model, logging_obj, etc. 

592 request: The originating FastAPI request, forwarded for disconnect handling 

593 

594 Returns: 

595 Async generator that yields SSE-formatted chat completion chunks 

596 """ 

597 # Any async-iterable here is a Responses API event stream needing conversion. 

598 # Class-identity checks miss router-wrapped streams (e.g. 

599 # HiddenParamsAsyncIteratorWrapper around LiteLLMCompletionStreamingIterator), 

600 # which previously leaked raw Responses events to the client. 

601 if hasattr(response, "__anext__"): 

602 # Transform Responses API iterator to chat completion iterator 

603 # Cast to AsyncIterator[str] since the stream implements __aiter__/__anext__ 

604 completion_stream: Final = responses_api_bridge.transformation_handler.get_model_response_iterator( 

605 streaming_response=cast(AsyncIterator[str], response), 

606 sync_stream=False, 

607 json_mode=False, 

608 ) 

609 # Wrap in CustomStreamWrapper to get the async generator 

610 logging_obj: Final = request_data.get("litellm_logging_obj") 

611 streamwrapper: Final = CustomStreamWrapper( 

612 completion_stream=completion_stream, 

613 model=request_data.get("model", ""), 

614 custom_llm_provider=None, 

615 logging_obj=logging_obj, 

616 ) 

617 # Use async_data_generator to format as SSE 

618 return async_data_generator( 

619 response=streamwrapper, 

620 user_api_key_dict=user_api_key_dict, 

621 request_data=request_data, 

622 request=request, 

623 ) 

624 # Otherwise, use the default generator 

625 return async_data_generator( 

626 response=response, 

627 user_api_key_dict=user_api_key_dict, 

628 request_data=request_data, 

629 request=request, 

630 ) 

631 

632 try: 

633 response: Final = await processor.base_process_llm_request( 

634 request=request, 

635 fastapi_response=fastapi_response, 

636 user_api_key_dict=user_api_key_dict, 

637 route_type="aresponses", 

638 proxy_logging_obj=proxy_logging_obj, 

639 llm_router=llm_router, 

640 general_settings=general_settings, 

641 proxy_config=proxy_config, 

642 select_data_generator=cursor_data_generator, 

643 model=None, 

644 user_model=user_model, 

645 user_temperature=user_temperature, 

646 user_request_timeout=user_request_timeout, 

647 user_max_tokens=user_max_tokens, 

648 user_api_base=user_api_base, 

649 version=version, 

650 ) 

651 

652 # Transform non-streaming Responses API response to chat completions format 

653 if isinstance(response, ResponsesAPIResponse): 

654 logging_obj: Final = processor.data.get("litellm_logging_obj") 

655 transformed_response: Final = responses_api_bridge.transformation_handler.transform_response( 

656 model=processor.data.get("model", ""), 

657 raw_response=response, 

658 model_response=ModelResponse(), 

659 logging_obj=cast(Any, logging_obj), 

660 request_data=processor.data, 

661 messages=processor.data.get("input", []), 

662 optional_params={}, 

663 litellm_params={}, 

664 encoding=None, 

665 api_key=None, 

666 json_mode=None, 

667 ) 

668 return transformed_response 

669 

670 # Streaming responses are already transformed by cursor_select_data_generator 

671 return response 

672 except Exception as e: 

673 raise await processor._handle_llm_api_exception( 

674 e=e, 

675 user_api_key_dict=user_api_key_dict, 

676 proxy_logging_obj=proxy_logging_obj, 

677 version=version, 

678 ) 

679 

680 

681@router.get( 

682 "/v1/responses/{response_id}", 

683 dependencies=[Depends(user_api_key_auth)], 

684 tags=["responses"], 

685 responses=RESPONSES_API_RESPONSE_SCHEMAS, 

686) 

687@router.get( 

688 "/responses/{response_id}", 

689 dependencies=[Depends(user_api_key_auth)], 

690 tags=["responses"], 

691 responses=RESPONSES_API_RESPONSE_SCHEMAS, 

692) 

693@router.get( 

694 "/openai/v1/responses/{response_id}", 

695 dependencies=[Depends(user_api_key_auth)], 

696 tags=["responses"], 

697 responses=RESPONSES_API_RESPONSE_SCHEMAS, 

698) 

699async def get_response( 

700 response_id: str, 

701 request: Request, 

702 fastapi_response: Response, 

703 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

704): 

705 """ 

706 Get a response by ID. 

707  

708 Supports both: 

709 - Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses 

710 - Provider response IDs: Passes through to provider API 

711  

712 Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get 

713  

714 ```bash 

715 # Get polling response 

716 curl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123 \ 

717 -H "Authorization: Bearer sk-1234" 

718  

719 # Get provider response 

720 curl -X GET http://localhost:4000/v1/responses/resp_abc123 \ 

721 -H "Authorization: Bearer sk-1234" 

722 ``` 

723 """ 

724 from litellm.proxy.proxy_server import ( 

725 _read_request_body, 

726 general_settings, 

727 llm_router, 

728 proxy_config, 

729 proxy_logging_obj, 

730 redis_usage_cache, 

731 select_data_generator, 

732 user_api_base, 

733 user_max_tokens, 

734 user_model, 

735 user_request_timeout, 

736 user_temperature, 

737 version, 

738 ) 

739 from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler 

740 

741 # Check if this is a polling ID 

742 if ResponsePollingHandler.is_polling_id(response_id): 742 ↛ 744line 742 didn't jump to line 744 because the condition on line 742 was never true

743 # Handle polling response 

744 if not redis_usage_cache: 

745 raise HTTPException( 

746 status_code=500, 

747 detail="Redis cache not configured. Polling requires Redis.", 

748 ) 

749 

750 polling_handler: Final = ResponsePollingHandler(redis_cache=redis_usage_cache) 

751 

752 # Get current state from cache 

753 state: Final = await polling_handler.get_state(response_id) 

754 

755 if not state: 

756 raise HTTPException( 

757 status_code=404, 

758 detail=f"Polling response {response_id} not found or expired", 

759 ) 

760 

761 # Return the whole state directly (OpenAI Response object format) 

762 # https://platform.openai.com/docs/api-reference/responses/object 

763 return state 

764 

765 # Normal provider response flow 

766 data: Final = await _read_request_body(request=request) 

767 data["response_id"] = response_id 

768 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

769 try: 

770 return await processor.base_process_llm_request( 

771 request=request, 

772 fastapi_response=fastapi_response, 

773 user_api_key_dict=user_api_key_dict, 

774 route_type="aget_responses", 

775 proxy_logging_obj=proxy_logging_obj, 

776 llm_router=llm_router, 

777 general_settings=general_settings, 

778 proxy_config=proxy_config, 

779 select_data_generator=select_data_generator, 

780 model=None, 

781 user_model=user_model, 

782 user_temperature=user_temperature, 

783 user_request_timeout=user_request_timeout, 

784 user_max_tokens=user_max_tokens, 

785 user_api_base=user_api_base, 

786 version=version, 

787 ) 

788 except Exception as e: 

789 raise await processor._handle_llm_api_exception( 

790 e=e, 

791 user_api_key_dict=user_api_key_dict, 

792 proxy_logging_obj=proxy_logging_obj, 

793 version=version, 

794 ) 

795 

796 

797@router.delete( 

798 "/v1/responses/{response_id}", 

799 dependencies=[Depends(user_api_key_auth)], 

800 tags=["responses"], 

801 responses=DELETE_RESPONSE_SCHEMAS, 

802) 

803@router.delete( 

804 "/responses/{response_id}", 

805 dependencies=[Depends(user_api_key_auth)], 

806 tags=["responses"], 

807 responses=DELETE_RESPONSE_SCHEMAS, 

808) 

809@router.delete( 

810 "/openai/v1/responses/{response_id}", 

811 dependencies=[Depends(user_api_key_auth)], 

812 tags=["responses"], 

813 responses=DELETE_RESPONSE_SCHEMAS, 

814) 

815async def delete_response( 

816 response_id: str, 

817 request: Request, 

818 fastapi_response: Response, 

819 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

820): 

821 """ 

822 Delete a response by ID. 

823  

824 Supports both: 

825 - Polling IDs (litellm_poll_*): Deletes from Redis cache 

826 - Provider response IDs: Passes through to provider API 

827  

828 Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/delete 

829  

830 ```bash 

831 curl -X DELETE http://localhost:4000/v1/responses/resp_abc123 \ 

832 -H "Authorization: Bearer sk-1234" 

833 ``` 

834 """ 

835 from litellm.proxy.proxy_server import ( 

836 _read_request_body, 

837 general_settings, 

838 llm_router, 

839 proxy_config, 

840 proxy_logging_obj, 

841 redis_usage_cache, 

842 select_data_generator, 

843 user_api_base, 

844 user_max_tokens, 

845 user_model, 

846 user_request_timeout, 

847 user_temperature, 

848 version, 

849 ) 

850 from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler 

851 

852 # Check if this is a polling ID 

853 if ResponsePollingHandler.is_polling_id(response_id): 853 ↛ 855line 853 didn't jump to line 855 because the condition on line 853 was never true

854 # Handle polling response deletion 

855 if not redis_usage_cache: 

856 raise HTTPException(status_code=500, detail="Redis cache not configured.") 

857 

858 polling_handler: Final = ResponsePollingHandler(redis_cache=redis_usage_cache) 

859 

860 # Get state to verify access 

861 state: Final = await polling_handler.get_state(response_id) 

862 

863 if not state: 

864 raise HTTPException(status_code=404, detail=f"Polling response {response_id} not found") 

865 

866 # Delete from cache 

867 success: Final = await polling_handler.delete_polling(response_id) 

868 

869 if success: 

870 return DeleteResponseResult(id=response_id, object="response", deleted=True) 

871 else: 

872 raise HTTPException(status_code=500, detail="Failed to delete polling response") 

873 

874 # Normal provider response flow 

875 data: Final = await _read_request_body(request=request) 

876 data["response_id"] = response_id 

877 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

878 try: 

879 return await processor.base_process_llm_request(