Coverage for .venv/lib/python3.13/site-packages/litellm/proxy/openai_evals_endpoints/endpoints.py: 86%

1""" 

2OpenAI Evals API endpoints - /v1/evals 

3""" 

4 

5from typing import Final 

6 

7import orjson 

8from fastapi import APIRouter, Depends, Request, Response 

9 

10from litellm.proxy._types import UserAPIKeyAuth 

11from litellm.proxy.auth.user_api_key_auth import user_api_key_auth 

12from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing 

13from litellm.types.llms.openai_evals import ( 

14 CancelEvalResponse, 

15 CancelRunResponse, 

16 DeleteEvalResponse, 

17 Eval, 

18 ListEvalsResponse, 

19 ListRunsResponse, 

20 Run, 

21 RunDeleteResponse, 

22) 

23 

24router: Final = APIRouter() 

25 

26 

27@router.post( 

28 "/v1/evals", 

29 tags=["OpenAI Evals API"], 

30 dependencies=[Depends(user_api_key_auth)], 

31 response_model=Eval, 

32) 

33async def create_eval( 

34 fastapi_response: Response, 

35 request: Request, 

36 custom_llm_provider: str | None = "openai", 

37 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

38) -> object: 

39 """ 

40 Create a new evaluation. 

41 

42 Model-based routing (for multi-account support): 

43 - Pass model via header: `x-litellm-model: gpt-4-account-1` 

44 - Pass model via query: `?model=gpt-4-account-1` 

45 - Pass model via body: `{"model": "gpt-4-account-1"}` 

46 

47 Example usage: 

48 ```bash 

49 curl -X POST "http://localhost:4000/v1/evals" \ 

50 -H "Authorization: Bearer your-key" \ 

51 -H "Content-Type: application/json" \ 

52 -d '{ 

53 "name": "Test Eval", 

54 "data_source_config": {"type": "file", "file_id": "file-abc123"}, 

55 "testing_criteria": {"graders": [{"type": "llm_as_judge"}]} 

56 }' 

57 ``` 

58 

59 Returns: Eval object with id, status, timestamps, etc. 

60 """ 

61 from litellm.proxy.proxy_server import ( 

62 general_settings, 

63 llm_router, 

64 proxy_config, 

65 proxy_logging_obj, 

66 select_data_generator, 

67 user_api_base, 

68 user_max_tokens, 

69 user_model, 

70 user_request_timeout, 

71 user_temperature, 

72 version, 

73 ) 

74 

75 # Read request body 

76 body: Final = await request.body() 

77 data: Final = orjson.loads(body) if body else {} 

78 

79 # Extract model for routing (header > query > body) 

80 # When using extra_body={"model": "..."}, the OpenAI SDK merges it into the body 

81 model: Final = data.get("model") or request.query_params.get("model") or request.headers.get("x-litellm-model") 

82 if model: 82 ↛ 83line 82 didn't jump to line 83 because the condition on line 82 was never true

83 data["model"] = model 

84 

85 if "custom_llm_provider" not in data: 85 ↛ 89line 85 didn't jump to line 89 because the condition on line 85 was always true

86 data["custom_llm_provider"] = custom_llm_provider 

87 

88 # Process request using ProxyBaseLLMRequestProcessing 

89 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

90 try: 

91 return await processor.base_process_llm_request( 

92 request=request, 

93 fastapi_response=fastapi_response, 

94 user_api_key_dict=user_api_key_dict, 

95 route_type="acreate_eval", 

96 proxy_logging_obj=proxy_logging_obj, 

97 llm_router=llm_router, 

98 general_settings=general_settings, 

99 proxy_config=proxy_config, 

100 select_data_generator=select_data_generator, 

101 model=data.get("model"), 

102 user_model=user_model, 

103 user_temperature=user_temperature, 

104 user_request_timeout=user_request_timeout, 

105 user_max_tokens=user_max_tokens, 

106 user_api_base=user_api_base, 

107 version=version, 

108 ) 

109 except Exception as e: 

110 raise await processor._handle_llm_api_exception( 

111 e=e, 

112 user_api_key_dict=user_api_key_dict, 

113 proxy_logging_obj=proxy_logging_obj, 

114 version=version, 

115 ) 

116 

117 

118@router.get( 

119 "/v1/evals", 

120 tags=["OpenAI Evals API"], 

121 dependencies=[Depends(user_api_key_auth)], 

122 response_model=ListEvalsResponse, 

123) 

124async def list_evals( 

125 fastapi_response: Response, 

126 request: Request, 

127 limit: int | None = 20, 

128 after: str | None = None, 

129 before: str | None = None, 

130 order: str | None = None, 

131 order_by: str | None = None, 

132 custom_llm_provider: str | None = "openai", 

133 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

134) -> object: 

135 """ 

136 List evaluations with pagination. 

137 

138 Model-based routing (for multi-account support): 

139 - Pass model via header: `x-litellm-model: gpt-4-account-1` 

140 - Pass model via query: `?model=gpt-4-account-1` 

141 - Pass model via body: `{"model": "gpt-4-account-1"}` 

142 

143 Example usage: 

144 ```bash 

145 curl "http://localhost:4000/v1/evals?limit=10" \ 

146 -H "Authorization: Bearer your-key" 

147 ``` 

148 

149 Returns: ListEvalsResponse with list of evaluations 

150 """ 

151 from litellm.proxy.proxy_server import ( 

152 general_settings, 

153 llm_router, 

154 proxy_config, 

155 proxy_logging_obj, 

156 select_data_generator, 

157 user_api_base, 

158 user_max_tokens, 

159 user_model, 

160 user_request_timeout, 

161 user_temperature, 

162 version, 

163 ) 

164 

165 # Read request body (optional for GET) 

166 body: Final = await request.body() 

167 data: Final = orjson.loads(body) if body else {} 

168 

169 # Use query params if not in body 

170 if "limit" not in data and limit is not None: 170 ↛ 172line 170 didn't jump to line 172 because the condition on line 170 was always true

171 data["limit"] = limit 

172 if "after" not in data and after is not None: 

173 data["after"] = after 

174 if "before" not in data and before is not None: 

175 data["before"] = before 

176 if "order" not in data and order is not None: 

177 data["order"] = order 

178 if "order_by" not in data and order_by is not None: 

179 data["order_by"] = order_by 

180 

181 # Extract model for routing (header > query > body) 

182 model: Final = data.get("model") or request.query_params.get("model") or request.headers.get("x-litellm-model") 

183 if model: 183 ↛ 184line 183 didn't jump to line 184 because the condition on line 183 was never true

184 data["model"] = model 

185 

186 if "custom_llm_provider" not in data: 186 ↛ 190line 186 didn't jump to line 190 because the condition on line 186 was always true

187 data["custom_llm_provider"] = custom_llm_provider 

188 

189 # Process request using ProxyBaseLLMRequestProcessing 

190 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

191 try: 

192 return await processor.base_process_llm_request( 

193 request=request, 

194 fastapi_response=fastapi_response, 

195 user_api_key_dict=user_api_key_dict, 

196 route_type="alist_evals", 

197 proxy_logging_obj=proxy_logging_obj, 

198 llm_router=llm_router, 

199 general_settings=general_settings, 

200 proxy_config=proxy_config, 

201 select_data_generator=select_data_generator, 

202 model=data.get("model"), 

203 user_model=user_model, 

204 user_temperature=user_temperature, 

205 user_request_timeout=user_request_timeout, 

206 user_max_tokens=user_max_tokens, 

207 user_api_base=user_api_base, 

208 version=version, 

209 ) 

210 except Exception as e: 

211 raise await processor._handle_llm_api_exception( 

212 e=e, 

213 user_api_key_dict=user_api_key_dict, 

214 proxy_logging_obj=proxy_logging_obj, 

215 version=version, 

216 ) 

217 

218 

219@router.get( 

220 "/v1/evals/{eval_id}", 

221 tags=["OpenAI Evals API"], 

222 dependencies=[Depends(user_api_key_auth)], 

223 response_model=Eval, 

224) 

225async def get_eval( 

226 eval_id: str, 

227 fastapi_response: Response, 

228 request: Request, 

229 custom_llm_provider: str | None = "openai", 

230 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

231) -> object: 

232 """ 

233 Get a specific evaluation by ID. 

234 

235 Model-based routing (for multi-account support): 

236 - Pass model via header: `x-litellm-model: gpt-4-account-1` 

237 - Pass model via query: `?model=gpt-4-account-1` 

238 - Pass model via body: `{"model": "gpt-4-account-1"}` 

239 

240 Example usage: 

241 ```bash 

242 curl "http://localhost:4000/v1/evals/eval_123" \ 

243 -H "Authorization: Bearer your-key" 

244 ``` 

245 

246 Returns: Eval object 

247 """ 

248 from litellm.proxy.proxy_server import ( 

249 general_settings, 

250 llm_router, 

251 proxy_config, 

252 proxy_logging_obj, 

253 select_data_generator, 

254 user_api_base, 

255 user_max_tokens, 

256 user_model, 

257 user_request_timeout, 

258 user_temperature, 

259 version, 

260 ) 

261 

262 # Read request body (optional for GET) 

263 body: Final = await request.body() 

264 data: Final = orjson.loads(body) if body else {} 

265 

266 # Set eval_id from path parameter 

267 data["eval_id"] = eval_id 

268 

269 # Extract model for routing (header > query > body) 

270 model: Final = data.get("model") or request.query_params.get("model") or request.headers.get("x-litellm-model") 

271 if model: 271 ↛ 272line 271 didn't jump to line 272 because the condition on line 271 was never true

272 data["model"] = model 

273 

274 if "custom_llm_provider" not in data: 274 ↛ 278line 274 didn't jump to line 278 because the condition on line 274 was always true

275 data["custom_llm_provider"] = custom_llm_provider 

276 

277 # Process request using ProxyBaseLLMRequestProcessing 

278 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

279 try: 

280 return await processor.base_process_llm_request( 

281 request=request, 

282 fastapi_response=fastapi_response, 

283 user_api_key_dict=user_api_key_dict, 

284 route_type="aget_eval", 

285 proxy_logging_obj=proxy_logging_obj, 

286 llm_router=llm_router, 

287 general_settings=general_settings, 

288 proxy_config=proxy_config, 

289 select_data_generator=select_data_generator, 

290 model=data.get("model"), 

291 user_model=user_model, 

292 user_temperature=user_temperature, 

293 user_request_timeout=user_request_timeout, 

294 user_max_tokens=user_max_tokens, 

295 user_api_base=user_api_base, 

296 version=version, 

297 ) 

298 except Exception as e: 

299 raise await processor._handle_llm_api_exception( 

300 e=e, 

301 user_api_key_dict=user_api_key_dict, 

302 proxy_logging_obj=proxy_logging_obj, 

303 version=version, 

304 ) 

305 

306 

307@router.post( 

308 "/v1/evals/{eval_id}", 

309 tags=["OpenAI Evals API"], 

310 dependencies=[Depends(user_api_key_auth)], 

311 response_model=Eval, 

312) 

313async def update_eval( 

314 eval_id: str, 

315 fastapi_response: Response, 

316 request: Request, 

317 custom_llm_provider: str | None = "openai", 

318 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

319) -> object: 

320 """ 

321 Update an evaluation. 

322 

323 Model-based routing (for multi-account support): 

324 - Pass model via header: `x-litellm-model: gpt-4-account-1` 

325 - Pass model via query: `?model=gpt-4-account-1` 

326 - Pass model via body: `{"model": "gpt-4-account-1"}` 

327 

328 Example usage: 

329 ```bash 

330 curl -X POST "http://localhost:4000/v1/evals/eval_123" \ 

331 -H "Authorization: Bearer your-key" \ 

332 -H "Content-Type: application/json" \ 

333 -d '{"name": "Updated Name"}' 

334 ``` 

335 

336 Returns: Updated Eval object 

337 """ 

338 from litellm.proxy.proxy_server import ( 

339 general_settings, 

340 llm_router, 

341 proxy_config, 

342 proxy_logging_obj, 

343 select_data_generator, 

344 user_api_base, 

345 user_max_tokens, 

346 user_model, 

347 user_request_timeout, 

348 user_temperature, 

349 version, 

350 ) 

351 

352 # Read request body 

353 body: Final = await request.body() 

354 data: Final = orjson.loads(body) if body else {} 

355 

356 # Set eval_id from path parameter 

357 data["eval_id"] = eval_id 

358 

359 # Extract model for routing (header > query > body) 

360 model: Final = data.get("model") or request.query_params.get("model") or request.headers.get("x-litellm-model") 

361 if model: 361 ↛ 362line 361 didn't jump to line 362 because the condition on line 361 was never true

362 data["model"] = model 

363 

364 if "custom_llm_provider" not in data: 364 ↛ 368line 364 didn't jump to line 368 because the condition on line 364 was always true

365 data["custom_llm_provider"] = custom_llm_provider 

366 

367 # Process request using ProxyBaseLLMRequestProcessing 

368 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

369 try: 

370 return await processor.base_process_llm_request( 

371 request=request, 

372 fastapi_response=fastapi_response, 

373 user_api_key_dict=user_api_key_dict, 

374 route_type="aupdate_eval", 

375 proxy_logging_obj=proxy_logging_obj, 

376 llm_router=llm_router, 

377 general_settings=general_settings, 

378 proxy_config=proxy_config, 

379 select_data_generator=select_data_generator, 

380 model=data.get("model"), 

381 user_model=user_model, 

382 user_temperature=user_temperature, 

383 user_request_timeout=user_request_timeout, 

384 user_max_tokens=user_max_tokens, 

385 user_api_base=user_api_base, 

386 version=version, 

387 ) 

388 except Exception as e: 

389 raise await processor._handle_llm_api_exception( 

390 e=e, 

391 user_api_key_dict=user_api_key_dict, 

392 proxy_logging_obj=proxy_logging_obj, 

393 version=version, 

394 ) 

395 

396 

397@router.delete( 

398 "/v1/evals/{eval_id}", 

399 tags=["OpenAI Evals API"], 

400 dependencies=[Depends(user_api_key_auth)], 

401 response_model=DeleteEvalResponse, 

402) 

403async def delete_eval( 

404 eval_id: str, 

405 fastapi_response: Response, 

406 request: Request, 

407 custom_llm_provider: str | None = "openai", 

408 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

409) -> object: 

410 """ 

411 Delete an evaluation. 

412 

413 Model-based routing (for multi-account support): 

414 - Pass model via header: `x-litellm-model: gpt-4-account-1` 

415 - Pass model via query: `?model=gpt-4-account-1` 

416 - Pass model via body: `{"model": "gpt-4-account-1"}` 

417 

418 Example usage: 

419 ```bash 

420 curl -X DELETE "http://localhost:4000/v1/evals/eval_123" \ 

421 -H "Authorization: Bearer your-key" 

422 ``` 

423 

424 Returns: DeleteEvalResponse with deletion confirmation 

425 """ 

426 from litellm.proxy.proxy_server import ( 

427 general_settings, 

428 llm_router, 

429 proxy_config, 

430 proxy_logging_obj, 

431 select_data_generator, 

432 user_api_base, 

433 user_max_tokens, 

434 user_model, 

435 user_request_timeout, 

436 user_temperature, 

437 version, 

438 ) 

439 

440 # Read request body (optional for DELETE) 

441 body: Final = await request.body() 

442 data: Final = orjson.loads(body) if body else {} 

443 

444 # Set eval_id from path parameter 

445 data["eval_id"] = eval_id 

446 

447 # Extract model for routing (header > query > body) 

448 model: Final = data.get("model") or request.query_params.get("model") or request.headers.get("x-litellm-model") 

449 if model: 449 ↛ 450line 449 didn't jump to line 450 because the condition on line 449 was never true

450 data["model"] = model 

451 

452 if "custom_llm_provider" not in data: 452 ↛ 456line 452 didn't jump to line 456 because the condition on line 452 was always true

453 data["custom_llm_provider"] = custom_llm_provider 

454 

455 # Process request using ProxyBaseLLMRequestProcessing 

456 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

457 try: 

458 return await processor.base_process_llm_request( 

459 request=request, 

460 fastapi_response=fastapi_response, 

461 user_api_key_dict=user_api_key_dict, 

462 route_type="adelete_eval", 

463 proxy_logging_obj=proxy_logging_obj, 

464 llm_router=llm_router, 

465 general_settings=general_settings, 

466 proxy_config=proxy_config, 

467 select_data_generator=select_data_generator, 

468 model=data.get("model"), 

469 user_model=user_model, 

470 user_temperature=user_temperature, 

471 user_request_timeout=user_request_timeout, 

472 user_max_tokens=user_max_tokens, 

473 user_api_base=user_api_base, 

474 version=version, 

475 ) 

476 except Exception as e: 

477 raise await processor._handle_llm_api_exception( 

478 e=e, 

479 user_api_key_dict=user_api_key_dict, 

480 proxy_logging_obj=proxy_logging_obj, 

481 version=version, 

482 ) 

483 

484 

485@router.post( 

486 "/v1/evals/{eval_id}/cancel", 

487 tags=["OpenAI Evals API"], 

488 dependencies=[Depends(user_api_key_auth)], 

489 response_model=CancelEvalResponse, 

490) 

491async def cancel_eval( 

492 eval_id: str, 

493 fastapi_response: Response, 

494 request: Request, 

495 custom_llm_provider: str | None = "openai", 

496 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

497) -> object: 

498 """ 

499 Cancel a running evaluation. 

500 

501 Model-based routing (for multi-account support): 

502 - Pass model via header: `x-litellm-model: gpt-4-account-1` 

503 - Pass model via query: `?model=gpt-4-account-1` 

504 - Pass model via body: `{"model": "gpt-4-account-1"}` 

505 

506 Example usage: 

507 ```bash 

508 curl -X POST "http://localhost:4000/v1/evals/eval_123/cancel" \ 

509 -H "Authorization: Bearer your-key" 

510 ``` 

511 

512 Returns: CancelEvalResponse with cancellation confirmation 

513 """ 

514 from litellm.proxy.proxy_server import ( 

515 general_settings, 

516 llm_router, 

517 proxy_config, 

518 proxy_logging_obj, 

519 select_data_generator, 

520 user_api_base, 

521 user_max_tokens, 

522 user_model, 

523 user_request_timeout, 

524 user_temperature, 

525 version, 

526 ) 

527 

528 # Read request body (optional for cancel) 

529 body: Final = await request.body() 

530 data: Final = orjson.loads(body) if body else {} 

531 

532 # Set eval_id from path parameter 

533 data["eval_id"] = eval_id 

534 

535 # Extract model for routing (header > query > body) 

536 model: Final = data.get("model") or request.query_params.get("model") or request.headers.get("x-litellm-model") 

537 if model: 537 ↛ 538line 537 didn't jump to line 538 because the condition on line 537 was never true

538 data["model"] = model 

539 

540 if "custom_llm_provider" not in data: 540 ↛ 544line 540 didn't jump to line 544 because the condition on line 540 was always true

541 data["custom_llm_provider"] = custom_llm_provider 

542 

543 # Process request using ProxyBaseLLMRequestProcessing 

544 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

545 try: 

546 return await processor.base_process_llm_request( 

547 request=request, 

548 fastapi_response=fastapi_response, 

549 user_api_key_dict=user_api_key_dict, 

550 route_type="acancel_eval", 

551 proxy_logging_obj=proxy_logging_obj, 

552 llm_router=llm_router, 

553 general_settings=general_settings, 

554 proxy_config=proxy_config, 

555 select_data_generator=select_data_generator, 

556 model=data.get("model"), 

557 user_model=user_model, 

558 user_temperature=user_temperature, 

559 user_request_timeout=user_request_timeout, 

560 user_max_tokens=user_max_tokens, 

561 user_api_base=user_api_base, 

562 version=version, 

563 ) 

564 except Exception as e: 

565 raise await processor._handle_llm_api_exception( 

566 e=e, 

567 user_api_key_dict=user_api_key_dict, 

568 proxy_logging_obj=proxy_logging_obj, 

569 version=version, 

570 ) 

571 

572 

573# =================================== 

574# Run API Endpoints 

575# =================================== 

576 

577 

578@router.post( 

579 "/v1/evals/{eval_id}/runs", 

580 tags=["OpenAI Evals API - Runs"], 

581 dependencies=[Depends(user_api_key_auth)], 

582 response_model=Run, 

583) 

584async def create_run( 

585 eval_id: str, 

586 fastapi_response: Response, 

587 request: Request, 

588 custom_llm_provider: str | None = "openai", 

589 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

590) -> object: 

591 """ 

592 Create a new run for an evaluation. 

593 

594 Model-based routing (for multi-account support): 

595 - Pass model via header: `x-litellm-model: gpt-4-account-1` 

596 - Pass model via query: `?model=gpt-4-account-1` 

597 - Pass model via body: `{"model": "gpt-4-account-1"}` 

598 - Pass model via completion.model: `{"completion": {"model": "gpt-4-account-1"}}` 

599 

600 Example usage: 

601 ```bash 

602 curl -X POST "http://localhost:4000/v1/evals/eval_123/runs" \ 

603 -H "Authorization: Bearer your-key" \ 

604 -H "Content-Type: application/json" \ 

605 -d '{ 

606 "data_source": {"type": "dataset", "dataset_id": "dataset_123"}, 

607 "completion": {"model": "gpt-4", "temperature": 0.7} 

608 }' 

609 ``` 

610 

611 Returns: Run object with id, status, timestamps, etc. 

612 """ 

613 from litellm.proxy.proxy_server import ( 

614 general_settings, 

615 llm_router, 

616 proxy_config, 

617 proxy_logging_obj, 

618 select_data_generator, 

619 user_api_base, 

620 user_max_tokens, 

621 user_model, 

622 user_request_timeout, 

623 user_temperature, 

624 version, 

625 ) 

626 

627 # Read request body 

628 body: Final = await request.body() 

629 data: Final = orjson.loads(body) if body else {} 

630 

631 # Set eval_id from path parameter 

632 data["eval_id"] = eval_id 

633 

634 # Extract model for routing (header > query > body > completion.model) 

635 model: Final = ( 

636 request.headers.get("x-litellm-model") 

637 or request.query_params.get("model") 

638 or data.get("model") 

639 or (data.get("completion", {}).get("model") if isinstance(data.get("completion"), dict) else None) 

640 ) 

641 if model: 641 ↛ 642line 641 didn't jump to line 642 because the condition on line 641 was never true

642 data["model"] = model 

643 

644 if "custom_llm_provider" not in data: 644 ↛ 648line 644 didn't jump to line 648 because the condition on line 644 was always true

645 data["custom_llm_provider"] = custom_llm_provider 

646 

647 # Process request using ProxyBaseLLMRequestProcessing 

648 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

649 try: 

650 return await processor.base_process_llm_request( 

651 request=request, 

652 fastapi_response=fastapi_response, 

653 user_api_key_dict=user_api_key_dict, 

654 route_type="acreate_run", 

655 proxy_logging_obj=proxy_logging_obj, 

656 llm_router=llm_router, 

657 general_settings=general_settings, 

658 proxy_config=proxy_config, 

659 select_data_generator=select_data_generator, 

660 model=data.get("model"), 

661 user_model=user_model, 

662 user_temperature=user_temperature, 

663 user_request_timeout=user_request_timeout, 

664 user_max_tokens=user_max_tokens, 

665 user_api_base=user_api_base, 

666 version=version, 

667 ) 

668 except Exception as e: 

669 raise await processor._handle_llm_api_exception( 

670 e=e, 

671 user_api_key_dict=user_api_key_dict, 

672 proxy_logging_obj=proxy_logging_obj, 

673 version=version, 

674 ) 

675 

676 

677@router.get( 

678 "/v1/evals/{eval_id}/runs", 

679 tags=["OpenAI Evals API - Runs"], 

680 dependencies=[Depends(user_api_key_auth)], 

681 response_model=ListRunsResponse, 

682) 

683async def list_runs( 

684 eval_id: str, 

685 fastapi_response: Response, 

686 request: Request, 

687 limit: int | None = 20, 

688 after: str | None = None, 

689 before: str | None = None, 

690 order: str | None = None, 

691 custom_llm_provider: str | None = "openai", 

692 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

693) -> object: 

694 """ 

695 List all runs for an evaluation with pagination. 

696 

697 Model-based routing (for multi-account support): 

698 - Pass model via header: `x-litellm-model: gpt-4-account-1` 

699 - Pass model via query: `?model=gpt-4-account-1` 

700 

701 Example usage: 

702 ```bash 

703 curl "http://localhost:4000/v1/evals/eval_123/runs?limit=10" \ 

704 -H "Authorization: Bearer your-key" 

705 ``` 

706 

707 Returns: ListRunsResponse with list of runs 

708 """ 

709 from litellm.proxy.proxy_server import ( 

710 general_settings, 

711 llm_router, 

712 proxy_config, 

713 proxy_logging_obj, 

714 select_data_generator, 

715 user_api_base, 

716 user_max_tokens, 

717 user_model, 

718 user_request_timeout, 

719 user_temperature, 

720 version, 

721 ) 

722 

723 # Build request data 

724 data: Final = { 

725 "eval_id": eval_id, 

726 "limit": limit, 

727 "after": after, 

728 "before": before, 

729 "order": order, 

730 } 

731 

732 # Extract model for routing (header > query) 

733 model: Final = request.headers.get("x-litellm-model") or request.query_params.get("model") 

734 if model: 734 ↛ 735line 734 didn't jump to line 735 because the condition on line 734 was never true

735 data["model"] = model 

736 

737 if "custom_llm_provider" not in data: 737 ↛ 741line 737 didn't jump to line 741 because the condition on line 737 was always true

738 data["custom_llm_provider"] = custom_llm_provider 

739 

740 # Process request using ProxyBaseLLMRequestProcessing 

741 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

742 try: 

743 return await processor.base_process_llm_request( 

744 request=request, 

745 fastapi_response=fastapi_response, 

746 user_api_key_dict=user_api_key_dict, 

747 route_type="alist_runs", 

748 proxy_logging_obj=proxy_logging_obj, 

749 llm_router=llm_router, 

750 general_settings=general_settings, 

751 proxy_config=proxy_config, 

752 select_data_generator=select_data_generator, 

753 model=str(data.get("model")) if data.get("model") is not None else None, 

754 user_model=user_model, 

755 user_temperature=user_temperature, 

756 user_request_timeout=user_request_timeout, 

757 user_max_tokens=user_max_tokens, 

758 user_api_base=user_api_base, 

759 version=version, 

760 ) 

761 except Exception as e: 

762 raise await processor._handle_llm_api_exception( 

763 e=e, 

764 user_api_key_dict=user_api_key_dict, 

765 proxy_logging_obj=proxy_logging_obj, 

766 version=version, 

767 ) 

768 

769 

770@router.get( 

771 "/v1/evals/{eval_id}/runs/{run_id}", 

772 tags=["OpenAI Evals API - Runs"], 

773 dependencies=[Depends(user_api_key_auth)], 

774 response_model=Run, 

775) 

776async def get_run( 

777 eval_id: str, 

778 run_id: str, 

779 fastapi_response: Response, 

780 request: Request, 

781 custom_llm_provider: str | None = "openai", 

782 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

783) -> object: 

784 """ 

785 Get a specific run by ID. 

786 

787 Model-based routing (for multi-account support): 

788 - Pass model via header: `x-litellm-model: gpt-4-account-1` 

789 - Pass model via query: `?model=gpt-4-account-1` 

790 

791 Example usage: 

792 ```bash 

793 curl "http://localhost:4000/v1/evals/eval_123/runs/run_456" \ 

794 -H "Authorization: Bearer your-key" 

795 ``` 

796 

797 Returns: Run object with full details 

798 """ 

799 from litellm.proxy.proxy_server import ( 

800 general_settings, 

801 llm_router, 

802 proxy_config, 

803 proxy_logging_obj, 

804 select_data_generator, 

805 user_api_base, 

806 user_max_tokens, 

807 user_model, 

808 user_request_timeout, 

809 user_temperature, 

810 version, 

811 ) 

812 

813 # Build request data 

814 data: Final = { 

815 "eval_id": eval_id, 

816 "run_id": run_id, 

817 } 

818 

819 # Extract model for routing (header > query) 

820 model: Final = request.headers.get("x-litellm-model") or request.query_params.get("model") 

821 if model: 821 ↛ 822line 821 didn't jump to line 822 because the condition on line 821 was never true

822 data["model"] = model 

823 

824 if "custom_llm_provider" not in data and custom_llm_provider is not None: 824 ↛ 828line 824 didn't jump to line 828 because the condition on line 824 was always true

825 data["custom_llm_provider"] = custom_llm_provider 

826 

827 # Process request using ProxyBaseLLMRequestProcessing 

828 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

829 try: 

830 return await processor.base_process_llm_request( 

831 request=request, 

832 fastapi_response=fastapi_response, 

833 user_api_key_dict=user_api_key_dict, 

834 route_type="aget_run", 

835 proxy_logging_obj=proxy_logging_obj, 

836 llm_router=llm_router, 

837 general_settings=general_settings, 

838 proxy_config=proxy_config, 

839 select_data_generator=select_data_generator, 

840 model=data.get("model"), 

841 user_model=user_model, 

842 user_temperature=user_temperature, 

843 user_request_timeout=user_request_timeout, 

844 user_max_tokens=user_max_tokens, 

845 user_api_base=user_api_base, 

846 version=version, 

847 ) 

848 except Exception as e: 

849 raise await processor._handle_llm_api_exception( 

850 e=e, 

851 user_api_key_dict=user_api_key_dict, 

852 proxy_logging_obj=proxy_logging_obj, 

853 version=version, 

854 ) 

855 

856 

857@router.post( 

858 "/v1/evals/{eval_id}/runs/{run_id}", 

859 tags=["OpenAI Evals API - Runs"], 

860 dependencies=[Depends(user_api_key_auth)], 

861 response_model=CancelRunResponse, 

862) 

863async def cancel_run( 

864 eval_id: str, 

865 run_id: str, 

866 fastapi_response: Response, 

867 request: Request, 

868 custom_llm_provider: str | None = "openai", 

869 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), 

870) -> object: 

871 """ 

872 Cancel a running run. 

873 

874 Model-based routing (for multi-account support): 

875 - Pass model via header: `x-litellm-model: gpt-4-account-1` 

876 - Pass model via query: `?model=gpt-4-account-1` 

877 

878 Example usage: 

879 ```bash 

880 curl -X POST "http://localhost:4000/v1/evals/eval_123/runs/run_456/cancel" \ 

881 -H "Authorization: Bearer your-key" 

882 ``` 

883 

884 Returns: CancelRunResponse with cancellation confirmation 

885 """ 

886 from litellm.proxy.proxy_server import ( 

887 general_settings, 

888 llm_router, 

889 proxy_config, 

890 proxy_logging_obj, 

891 select_data_generator, 

892 user_api_base, 

893 user_max_tokens, 

894 user_model, 

895 user_request_timeout, 

896 user_temperature, 

897 version, 

898 ) 

899 

900 # Read request body (optional for cancel) 

901 body: Final = await request.body() 

902 data: Final = orjson.loads(body) if body else {} 

903 

904 # Set eval_id and run_id from path parameters 

905 data["eval_id"] = eval_id 

906 data["run_id"] = run_id 

907 

908 # Extract model for routing (header > query > body) 

909 model: Final = data.get("model") or request.query_params.get("model") or request.headers.get("x-litellm-model") 

910 if model: 910 ↛ 911line 910 didn't jump to line 911 because the condition on line 910 was never true

911 data["model"] = model 

912 

913 if "custom_llm_provider" not in data: 913 ↛ 917line 913 didn't jump to line 917 because the condition on line 913 was always true

914 data["custom_llm_provider"] = custom_llm_provider 

915 

916 # Process request using ProxyBaseLLMRequestProcessing 

917 processor: Final = ProxyBaseLLMRequestProcessing(data=data) 

918 try: 

919 return await processor.base_process_llm_request( 

920 request=request, 

921 fastapi_response=fastapi_response, 

922 user_api_key_dict=user_api_key_dict, 

923 route_type="acancel_run", 

924 proxy_logging_obj=proxy_logging_obj, 

925 llm_router=llm_router, 

926 general_settings=general_settings, 

927 proxy_config=proxy_config, 

928 select_data_generator=select_data_generator, 

929 model=data.get("model"), 

930 user_model=user_model, 

931 user_temperature=user_temperature, 

932 user_request_timeout=user_request_timeout,