Coverage for open_webui/routers/ollama.py: 56%
864 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-07 05:07 +0000
1from __future__ import annotations
3import asyncio
4import logging
5import os
6import random
7import re
8import time
9from datetime import datetime
10from typing import Optional, Union
11from urllib.parse import urlparse
13import aiofiles
14import aiohttp
15from aiocache import cached
16from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
17from fastapi.responses import StreamingResponse
18from open_webui.config import UPLOAD_DIR
19from open_webui.constants import ERROR_MESSAGES
20from open_webui.env import (
21 AIOHTTP_CLIENT_SESSION_SSL,
22 AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST,
23 AIOHTTP_FILE_STREAM_CHUNK_SIZE,
24 BYPASS_MODEL_ACCESS_CONTROL,
25 ENABLE_FORWARD_USER_INFO_HEADERS,
26 FORWARD_SESSION_INFO_HEADER_CHAT_ID,
27 MODELS_CACHE_TTL,
28 REDIS_KEY_PREFIX,
29)
30from open_webui.events import EVENTS, publish_event, publish_model_provider_request_failed
31from open_webui.internal.db import get_async_session
32from open_webui.models.access_grants import AccessGrants
33from open_webui.models.config import Config
34from open_webui.models.groups import Groups
35from open_webui.models.models import Models
36from open_webui.models.users import UserModel
37from open_webui.utils.access_control import check_model_access
38from open_webui.utils.auth import get_admin_user, get_verified_user
39from open_webui.utils.headers import get_custom_headers, include_user_info_headers
40from open_webui.utils.json_codec import JSONCodec
41from open_webui.utils.misc import calculate_sha256
42from open_webui.utils.model_ids import strip_provider_model_prefix
43from open_webui.utils.payload import (
44 apply_model_params_to_body_ollama,
45 apply_model_params_to_body_openai,
46 apply_system_prompt_to_body,
47)
48from open_webui.utils.session_pool import cleanup_response, get_client_timeout, get_session, stream_wrapper
49from pydantic import BaseModel, ConfigDict, validator
50from sqlalchemy.ext.asyncio import AsyncSession
52log = logging.getLogger(__name__)
54# Headers that become stale after aiohttp auto-decompresses the upstream
55# response body. Forwarding them verbatim causes desktop / programmatic
56# clients to attempt decompression of an already-decoded payload, resulting
57# in ZlibError. See https://github.com/aio-libs/aiohttp/issues/4462.
58# Also drop server and date: uvicorn adds its own and forwarding both duplicates them.
59_STRIP_PROXY_HEADERS = frozenset({'content-encoding', 'content-length', 'transfer-encoding', 'server', 'date'})
60_MODEL_LIST_TIMEOUT = aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST)
61BASE_MODELS_CACHE_KEY = f'{REDIS_KEY_PREFIX}:models:base'
64def _clean_proxy_headers(raw_headers) -> dict:
65 """Return a copy of *raw_headers* without the encoding, server and date headers."""
66 return {k: v for k, v in raw_headers.items() if k.lower() not in _STRIP_PROXY_HEADERS}
69async def send_get_request(
70 url: str,
71 key: str | None = None,
72 user: UserModel | None = None,
73):
74 """Issue a GET request to an Ollama backend and return JSON, or *None* on failure."""
75 try:
76 session = await get_session()
77 headers: dict = {
78 'Content-Type': 'application/json',
79 }
80 if key: 80 ↛ 81line 80 didn't jump to line 81 because the condition on line 80 was never true
81 headers['Authorization'] = f'Bearer {key}'
82 if ENABLE_FORWARD_USER_INFO_HEADERS and user: 82 ↛ 83line 82 didn't jump to line 83 because the condition on line 82 was never true
83 headers = include_user_info_headers(headers, user)
85 async with session.get(
86 url,
87 headers=headers,
88 ssl=AIOHTTP_CLIENT_SESSION_SSL,
89 timeout=_MODEL_LIST_TIMEOUT,
90 ) as r:
91 return await r.json(loads=JSONCodec.loads)
92 except Exception as exc:
93 log.error(f'Connection error: {exc}')
94 return None
97async def send_request(
98 url: str,
99 method: str = 'POST',
100 *,
101 payload: Union[str, bytes | None] = None,
102 key: str | None = None,
103 user: UserModel = None,
104 stream: bool = False,
105 # passthrough must stay False for /api/chat: middleware parses it per line
106 passthrough: bool = False,
107 content_type: str | None = None,
108 metadata: dict | None = None,
109 api_config: dict | None = None,
110 request: Request | None = None,
111):
112 r = None
113 streaming = False
114 try:
115 session = await get_session()
117 headers = {
118 'Content-Type': 'application/json',
119 **({'Authorization': f'Bearer {key}'} if key else {}),
120 }
122 if ENABLE_FORWARD_USER_INFO_HEADERS and user: 122 ↛ 123line 122 didn't jump to line 123 because the condition on line 122 was never true
123 headers = include_user_info_headers(headers, user, request=request)
124 if metadata and metadata.get('chat_id'):
125 headers[FORWARD_SESSION_INFO_HEADER_CHAT_ID] = metadata.get('chat_id')
127 # Custom per-connection headers last so admin-set headers take precedence.
128 if api_config and api_config.get('headers'): 128 ↛ 129line 128 didn't jump to line 129 because the condition on line 128 was never true
129 headers.update(await get_custom_headers(api_config['headers'], user, metadata, request=request))
131 r = await session.request(
132 method,
133 url,
134 data=payload,
135 headers=headers,
136 ssl=AIOHTTP_CLIENT_SESSION_SSL,
137 timeout=get_client_timeout(stream=stream),
138 )
140 if not r.ok:
141 try:
142 res = await r.json(loads=JSONCodec.loads)
143 await publish_model_provider_request_failed(
144 request,
145 actor=user,
146 provider='ollama',
147 base_url=url,
148 status=r.status,
149 upstream_error=res,
150 )
151 if 'error' in res:
152 raise HTTPException(status_code=r.status, detail=res['error'])
153 except HTTPException:
154 raise
155 except Exception as e:
156 log.error(f'Failed to parse error response: {e}')
157 await publish_model_provider_request_failed(
158 request,
159 actor=user,
160 provider='ollama',
161 base_url=url,
162 status=r.status,
163 )
164 raise HTTPException(
165 status_code=r.status,
166 detail=ERROR_MESSAGES.SERVER_CONNECTION_ERROR,
167 )
169 r.raise_for_status()
171 if stream:
172 response_headers = _clean_proxy_headers(r.headers)
173 if content_type:
174 response_headers['Content-Type'] = content_type
176 streaming = True
177 return StreamingResponse(
178 stream_wrapper(r, passthrough=passthrough),
179 status_code=r.status,
180 headers=response_headers,
181 )
182 else:
183 try:
184 return await r.json(loads=JSONCodec.loads)
185 except Exception:
186 return None
188 except HTTPException:
189 raise
190 except Exception as e:
191 raise HTTPException(
192 status_code=r.status if r else 500,
193 detail=f'Ollama: {e}' if str(e) else ERROR_MESSAGES.SERVER_CONNECTION_ERROR,
194 )
195 finally:
196 if not streaming:
197 await cleanup_response(r)
200def get_api_key(idx, url, configs):
201 parsed_url = urlparse(url)
202 base_url = f'{parsed_url.scheme}://{parsed_url.netloc}'
203 return configs.get(str(idx), configs.get(base_url, {})).get('key', None) # Legacy support
206##########################################
207#
208# API routes
209#
210##########################################
212router = APIRouter()
214OLLAMA_CONFIG_KEYS = {
215 'ENABLE_OLLAMA_API': 'ollama.enable',
216 'OLLAMA_BASE_URLS': 'ollama.base_urls',
217 'OLLAMA_API_CONFIGS': 'ollama.api_configs',
218}
221async def get_ollama_config_values() -> dict:
222 values = await Config.get_many(*OLLAMA_CONFIG_KEYS.values())
223 return {field: values[storage_key] for field, storage_key in OLLAMA_CONFIG_KEYS.items() if storage_key in values}
226async def get_ollama_runtime_config() -> tuple[bool, list[str], dict]:
227 values = await Config.get_many('ollama.enable', 'ollama.base_urls', 'ollama.api_configs')
228 return (
229 values.get('ollama.enable'),
230 values.get('ollama.base_urls') or [],
231 values.get('ollama.api_configs') or {},
232 )
235async def get_ollama_connection(idx: int) -> tuple[str, dict, str | None]:
236 _, base_urls, api_configs = await get_ollama_runtime_config()
237 url = base_urls[idx]
238 return url, resolve_api_config(api_configs, idx, url), get_api_key(idx, url, api_configs)
241@router.head('/')
242@router.get('/')
243async def get_status() -> dict:
244 """Health-check endpoint."""
245 return {'status': True}
248class ConnectionVerificationForm(BaseModel):
249 url: str
250 key: str | None = None
253@router.post('/verify')
254async def verify_connection(
255 form_data: ConnectionVerificationForm,
256 user=Depends(get_admin_user),
257):
258 """Verify that an Ollama backend at *form_data.url* is reachable."""
259 try:
260 session = await get_session()
261 headers: dict = {}
262 if form_data.key:
263 headers['Authorization'] = f'Bearer {form_data.key}'
264 if ENABLE_FORWARD_USER_INFO_HEADERS and user: 264 ↛ 265line 264 didn't jump to line 265 because the condition on line 264 was never true
265 headers = include_user_info_headers(headers, user)
267 async with session.get(
268 f'{form_data.url}/api/version',
269 headers=headers,
270 ssl=AIOHTTP_CLIENT_SESSION_SSL,
271 timeout=_MODEL_LIST_TIMEOUT,
272 ) as r:
273 if r.status != 200:
274 detail = f'HTTP Error: {r.status}'
275 res = await r.json(loads=JSONCodec.loads)
276 if 'error' in res:
277 detail = f'External Error: {res["error"]}'
278 raise Exception(detail)
280 return await r.json(loads=JSONCodec.loads)
281 except aiohttp.ClientError as exc:
282 log.exception(f'Client error: {exc}')
283 raise HTTPException(status_code=500, detail=ERROR_MESSAGES.SERVER_CONNECTION_ERROR)
284 except Exception as exc:
285 log.exception(f'Unexpected error: {exc}')
286 raise HTTPException(status_code=500, detail=f'Unexpected error: {exc}')
289@router.get('/config')
290async def get_config(
291 request: Request,
292 user=Depends(get_admin_user),
293) -> dict:
294 """Return the current Ollama connection configuration."""
295 return await get_ollama_config_values()
298class OllamaConfigForm(BaseModel):
299 """Payload for updating the Ollama connection configuration."""
301 ENABLE_OLLAMA_API: bool | None = None
302 OLLAMA_BASE_URLS: list[str]
303 OLLAMA_API_CONFIGS: dict
306@router.post('/config/update')
307async def update_config(
308 request: Request,
309 form_data: OllamaConfigForm,
310 user=Depends(get_admin_user),
311) -> dict:
312 """Persist updated Ollama connection settings."""
313 valid_keys = {str(i) for i in range(len(form_data.OLLAMA_BASE_URLS))}
314 api_configs = {k: v for k, v in form_data.OLLAMA_API_CONFIGS.items() if k in valid_keys}
316 await Config.upsert(
317 {
318 'ollama.enable': form_data.ENABLE_OLLAMA_API,
319 'ollama.base_urls': form_data.OLLAMA_BASE_URLS,
320 'ollama.api_configs': api_configs,
321 }
322 )
324 await get_all_models.cache.clear()
325 redis = getattr(request.app.state, 'redis', None)
326 if redis is not None: 326 ↛ 327line 326 didn't jump to line 327 because the condition on line 326 was never true
327 await redis.delete(BASE_MODELS_CACHE_KEY)
328 request.app.state.BASE_MODELS = []
329 request.app.state.OLLAMA_MODELS = {}
330 models = getattr(request.app.state, 'MODELS', None)
331 if hasattr(models, 'clear'): 331 ↛ 334line 331 didn't jump to line 334 because the condition on line 331 was always true
332 models.clear()
333 else:
334 request.app.state.MODELS = {}
336 await publish_event(
337 request,
338 EVENTS.MODEL_PROVIDER_CONFIG_UPDATED,
339 actor=user,
340 subject_id='ollama',
341 subject_type='model.provider_config',
342 data={
343 'provider': 'ollama',
344 'enabled': form_data.ENABLE_OLLAMA_API,
345 'base_url_count': len(form_data.OLLAMA_BASE_URLS),
346 },
347 )
348 return {
349 'ENABLE_OLLAMA_API': form_data.ENABLE_OLLAMA_API,
350 'OLLAMA_BASE_URLS': form_data.OLLAMA_BASE_URLS,
351 'OLLAMA_API_CONFIGS': api_configs,
352 }
355def merge_models_lists(model_lists) -> list[dict]:
356 """De-duplicate model entries across multiple Ollama backends, tracking which URL index hosts each model."""
357 merged: dict[str, dict] = {}
358 for idx, entries in enumerate(model_lists):
359 if entries is None: 359 ↛ 361line 359 didn't jump to line 361 because the condition on line 359 was always true
360 continue
361 for entry in entries:
362 model_id = entry.get('model')
363 if model_id is None:
364 continue
365 if model_id not in merged:
366 entry['urls'] = [idx]
367 merged[model_id] = entry
368 else:
369 merged[model_id]['urls'].append(idx)
370 return list(merged.values())
373def resolve_api_config(api_configs: dict, idx: int, url: str) -> dict:
374 """Look up the API config for a backend by numeric index, falling back to URL key (legacy)."""
375 return api_configs.get(str(idx), api_configs.get(url, {}))
378async def get_ollama_connection_config() -> tuple[list, dict]:
379 """Base URLs and per-connection API configs in one batched SELECT."""
380 config = await Config.get_many('ollama.base_urls', 'ollama.api_configs')
381 return config.get('ollama.base_urls', []), config.get('ollama.api_configs', {})
384@cached(
385 ttl=MODELS_CACHE_TTL,
386 # key_builder (not key) is the per-call hook in aiocache 0.12; `key=` is a
387 # static key, so a `key=lambda` collapsed every caller to one shared entry.
388 key_builder=lambda _func, request, user=None: f'ollama_all_models_{user.id}' if user else 'ollama_all_models',
389)
390async def get_all_models(request: Request, user: UserModel | None = None):
391 """Aggregate model tags from every enabled Ollama backend."""
392 log.info('get_all_models()')
394 if not await Config.get('ollama.enable'):
395 models_dict: dict = {'models': []}
396 request.app.state.OLLAMA_MODELS = {}
397 return models_dict
399 # Fan-out tag requests to every backend
400 tasks = []
401 base_urls = await Config.get('ollama.base_urls', [])
402 api_configs = await Config.get('ollama.api_configs', {})
403 for idx, url in enumerate(base_urls):
404 api_config = resolve_api_config(api_configs, idx, url)
405 if not api_config: 405 ↛ 407line 405 didn't jump to line 407 because the condition on line 405 was always true
406 tasks.append(send_get_request(f'{url}/api/tags', user=user))
407 elif api_config.get('enable', True):
408 tasks.append(send_get_request(f'{url}/api/tags', api_config.get('key'), user=user))
409 else:
410 tasks.append(asyncio.ensure_future(asyncio.sleep(0, None)))
412 responses = await asyncio.gather(*tasks)
414 # Track which backends failed so we can skip them for /api/ps
415 failed_idxs: set[int] = set()
417 # Post-process each response: apply prefix_id, tags, model filtering
418 for idx, response in enumerate(responses):
419 if not response: 419 ↛ 422line 419 didn't jump to line 422 because the condition on line 419 was always true
420 failed_idxs.add(idx)
421 continue
422 url = base_urls[idx]
423 api_config = resolve_api_config(api_configs, idx, url)
425 connection_type = api_config.get('connection_type', 'local')
426 prefix_id = api_config.get('prefix_id')
427 allowed_tags = api_config.get('tags', [])
428 allowed_model_ids = api_config.get('model_ids', [])
430 if allowed_model_ids and 'models' in response:
431 response['models'] = [m for m in response['models'] if m['model'] in allowed_model_ids]
433 for m in response.get('models', []):
434 if prefix_id:
435 m['model'] = f'{prefix_id}.{m["model"]}'
436 if m.get('name'):
437 m['name'] = f'{prefix_id}.{m["name"]}'
438 if allowed_tags:
439 m['tags'] = allowed_tags
440 if connection_type:
441 m['connection_type'] = connection_type
443 models_dict = {'models': merge_models_lists(r.get('models', []) if r else None for r in responses)}
445 # Annotate with expiry info from loaded-model state
446 try:
447 loaded = await get_ollama_loaded_models(request, user=user, skip_idxs=failed_idxs)
448 expires_map = {m['model']: m['expires_at'] for m in loaded['models'] if 'expires_at' in m}
449 for m in models_dict['models']: 449 ↛ 450line 449 didn't jump to line 450 because the loop on line 449 never started
450 if m['model'] in expires_map:
451 dt = datetime.fromisoformat(expires_map[m['model']])
452 m['expires_at'] = int(dt.timestamp())
453 except Exception as exc:
454 log.debug('Failed to get loaded models: %s', exc)
456 request.app.state.OLLAMA_MODELS = {m['model']: m for m in models_dict['models']}
457 return models_dict
460async def get_filtered_models(models, user, db=None):
461 """Return only the models the given *user* is allowed to access."""
462 model_ids = [m['model'] for m in models.get('models', [])]
463 model_infos = {mi.id: mi for mi in await Models.get_models_by_ids(model_ids, db=db)}
464 user_group_ids = {g.id for g in await Groups.get_groups_by_member_id(user.id, db=db)}
466 accessible_ids = await AccessGrants.get_accessible_resource_ids(
467 user_id=user.id,
468 resource_type='model',
469 resource_ids=list(model_infos.keys()),
470 permission='read',
471 user_group_ids=user_group_ids,
472 db=db,
473 )
474 return [
475 m
476 for m in models.get('models', [])
477 if (mi := model_infos.get(m['model'])) and (user.id == mi.user_id or mi.id in accessible_ids)
478 ]
481@router.get('/api/tags')
482@router.get('/api/tags/{url_idx}')
483async def get_ollama_tags(
484 request: Request,
485 url_idx: int | None = None,
486 user=Depends(get_verified_user),
487):
488 """List Ollama model tags, optionally from a specific backend."""
489 if url_idx is not None and user.role != 'admin': 489 ↛ 490line 489 didn't jump to line 490 because the condition on line 489 was never true
490 raise HTTPException(status_code=401, detail=ERROR_MESSAGES.ACCESS_PROHIBITED)
492 if not await Config.get('ollama.enable'): 492 ↛ 495line 492 didn't jump to line 495 because the condition on line 492 was always true
493 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
495 if url_idx is None:
496 result = await get_all_models(request, user=user)
497 else:
498 url = (await Config.get('ollama.base_urls', []))[url_idx]
499 key = get_api_key(url_idx, url, (await Config.get('ollama.api_configs', {})))
500 result = await send_request(f'{url}/api/tags', 'GET', key=key, user=user)
502 if user.role == 'user' and not BYPASS_MODEL_ACCESS_CONTROL:
503 result['models'] = await get_filtered_models(result, user)
505 return result
508@router.get('/api/ps')
509async def get_ollama_loaded_models(
510 request: Request,
511 user=Depends(get_admin_user),
512 skip_idxs: set[int] | None = None,
513) -> dict:
514 """List models currently loaded in Ollama memory across all backends."""
515 if not await Config.get('ollama.enable'):
516 return {'models': []}
518 tasks = []
519 base_urls = await Config.get('ollama.base_urls', [])
520 api_configs = await Config.get('ollama.api_configs', {})
521 for idx, url in enumerate(base_urls):
522 if skip_idxs and idx in skip_idxs: 522 ↛ 525line 522 didn't jump to line 525 because the condition on line 522 was always true
523 tasks.append(asyncio.ensure_future(asyncio.sleep(0, None)))
524 continue
525 api_config = resolve_api_config(api_configs, idx, url)
526 if not api_config:
527 tasks.append(send_get_request(f'{url}/api/ps', user=user))
528 elif api_config.get('enable', True):
529 tasks.append(send_get_request(f'{url}/api/ps', api_config.get('key'), user=user))
530 else:
531 tasks.append(asyncio.ensure_future(asyncio.sleep(0, None)))
533 responses = await asyncio.gather(*tasks)
535 for idx, response in enumerate(responses):
536 if not response: 536 ↛ 538line 536 didn't jump to line 538 because the condition on line 536 was always true
537 continue
538 api_config = resolve_api_config(api_configs, idx, base_urls[idx])
539 prefix_id = api_config.get('prefix_id')
540 if prefix_id:
541 for m in response.get('models', []):
542 m['model'] = f'{prefix_id}.{m["model"]}'
544 return {'models': merge_models_lists(r.get('models', []) if r else None for r in responses)}
547@router.get('/api/version')
548@router.get('/api/version/{url_idx}')
549async def get_ollama_versions(
550 request: Request,
551 user=Depends(get_verified_user),
552 url_idx: int | None = None,
553):
554 """Return the lowest Ollama version across all configured backends."""
555 if url_idx is not None and user.role != 'admin': 555 ↛ 556line 555 didn't jump to line 556 because the condition on line 555 was never true
556 raise HTTPException(status_code=401, detail=ERROR_MESSAGES.ACCESS_PROHIBITED)
558 if not await Config.get('ollama.enable'):
559 return {'version': False}
561 if url_idx is not None:
562 url = (await Config.get('ollama.base_urls', []))[url_idx]
563 return await send_request(f'{url}/api/version', 'GET')
565 # Fan-out to every enabled backend
566 tasks = []
567 for idx, url in enumerate(await Config.get('ollama.base_urls', [])):
568 api_config = (await Config.get('ollama.api_configs', {})).get(
569 str(idx),
570 (await Config.get('ollama.api_configs', {})).get(url, {}),
571 )
572 if api_config.get('enable', True): 572 ↛ 567line 572 didn't jump to line 567 because the condition on line 572 was always true
573 tasks.append(send_get_request(f'{url}/api/version', api_config.get('key')))
575 raw = await asyncio.gather(*tasks)
576 valid = [r for r in raw if r is not None]
578 if not valid: 578 ↛ 581line 578 didn't jump to line 581 because the condition on line 578 was always true
579 raise HTTPException(status_code=500, detail=ERROR_MESSAGES.OLLAMA_NOT_FOUND)
581 lowest = min(
582 valid,
583 key=lambda v: tuple(map(int, re.sub(r'^v|-.*', '', v['version']).split('.'))),
584 )
585 return {'version': lowest['version']}
588class ModelNameForm(BaseModel):
589 """Generic form carrying an optional model identifier."""
591 model: str | None = None
592 model_config = ConfigDict(extra='allow')
595@router.post('/api/unload')
596async def unload_model(
597 request: Request,
598 form_data: ModelNameForm,
599 user=Depends(get_admin_user),
600):
601 form_data = form_data.model_dump(exclude_none=True)
602 model = form_data.get('model', form_data.get('name'))
604 if not model:
605 raise HTTPException(status_code=400, detail='Missing name of the model to unload.')
607 # Refresh/load models if needed, get mapping from name to URLs
608 await get_all_models(request, user=user)
609 models = request.app.state.OLLAMA_MODELS
611 if model not in models: 611 ↛ 613line 611 didn't jump to line 613 because the condition on line 611 was always true
612 raise HTTPException(status_code=400, detail=ERROR_MESSAGES.MODEL_NOT_FOUND(model))
613 url_indices = models[model]['urls']
615 # Send unload to ALL url_indices
616 results = []
617 errors = []
618 for idx in url_indices:
619 url = (await Config.get('ollama.base_urls', []))[idx]
620 api_config = (await Config.get('ollama.api_configs', {})).get(
621 str(idx), (await Config.get('ollama.api_configs', {})).get(url, {})
622 )
623 key = get_api_key(idx, url, (await Config.get('ollama.api_configs', {})))
625 prefix_id = api_config.get('prefix_id', None)
626 if prefix_id and model.startswith(f'{prefix_id}.'):
627 model = model[len(f'{prefix_id}.') :]
629 payload = {'model': model, 'keep_alive': 0, 'prompt': ''}
631 try:
632 res = await send_request(
633 f'{url}/api/generate',
634 payload=JSONCodec.dumps(payload),
635 key=key,
636 user=user,
637 )
638 results.append({'url_idx': idx, 'success': True, 'response': res})
639 except Exception as e:
640 log.exception(f'Failed to unload model on node {idx}: {e}')
641 errors.append({'url_idx': idx, 'success': False, 'error': str(e)})
643 if len(errors) > 0:
644 raise HTTPException(
645 status_code=500,
646 detail=f'Failed to unload model on {len(errors)} nodes: {errors}',
647 )
649 return {'status': True}
652@router.post('/api/pull')
653@router.post('/api/pull/{url_idx}')
654async def pull_model(
655 request: Request,
656 form_data: ModelNameForm,
657 url_idx: int = 0,
658 user=Depends(get_admin_user),
659):
660 if not await Config.get('ollama.enable'):
661 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
663 form_data = form_data.model_dump(exclude_none=True)
664 form_data['model'] = form_data.get('model', form_data.get('name'))
666 url = (await Config.get('ollama.base_urls', []))[url_idx]
667 log.info('url: %s', url)
669 # Admins may pull from any registry
670 return await send_request(
671 f'{url}/api/pull',
672 payload=JSONCodec.dumps({**form_data, 'insecure': True}),
673 key=get_api_key(url_idx, url, (await Config.get('ollama.api_configs', {}))),
674 user=user,
675 stream=True,
676 passthrough=True,
677 )
680class PushModelForm(BaseModel):
681 """Payload for pushing a model to a registry."""
683 model: str
684 insecure: bool | None = None
685 stream: bool | None = None
688@router.delete('/api/push')
689@router.delete('/api/push/{url_idx}')
690async def push_model(
691 request: Request,
692 form_data: PushModelForm,
693 url_idx: int | None = None,
694 user=Depends(get_admin_user),
695):
696 """Push a local model to a remote registry."""
697 if not await Config.get('ollama.enable'):
698 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
700 if url_idx is None: 700 ↛ 707line 700 didn't jump to line 707 because the condition on line 700 was always true
701 await get_all_models(request, user=user)
702 models = request.app.state.OLLAMA_MODELS
703 if form_data.model not in models: 703 ↛ 705line 703 didn't jump to line 705 because the condition on line 703 was always true
704 raise HTTPException(status_code=400, detail=ERROR_MESSAGES.MODEL_NOT_FOUND(form_data.model))
705 url_idx = models[form_data.model]['urls'][0]
707 url = (await Config.get('ollama.base_urls', []))[url_idx]
708 log.debug('url: %s', url)
710 return await send_request(
711 f'{url}/api/push',
712 payload=form_data.model_dump_json(exclude_none=True).encode(),
713 key=get_api_key(url_idx, url, (await Config.get('ollama.api_configs', {}))),
714 user=user,
715 stream=True,
716 passthrough=True,
717 )
720class CreateModelForm(BaseModel):
721 """Payload for creating a new model via Modelfile."""
723 model: str | None = None
724 stream: bool | None = None
725 path: str | None = None
726 model_config = ConfigDict(extra='allow')
729@router.post('/api/create')
730@router.post('/api/create/{url_idx}')
731async def create_model(
732 request: Request,
733 form_data: CreateModelForm,
734 url_idx: int = 0,
735 user=Depends(get_admin_user),
736):
737 if not await Config.get('ollama.enable'):
738 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
740 log.debug('form_data: %s', form_data)
741 url = (await Config.get('ollama.base_urls', []))[url_idx]
743 return await send_request(
744 f'{url}/api/create',
745 payload=form_data.model_dump_json(exclude_none=True).encode(),
746 key=get_api_key(url_idx, url, (await Config.get('ollama.api_configs', {}))),
747 user=user,
748 stream=True,
749 passthrough=True,
750 )
753class CopyModelForm(BaseModel):
754 """Payload for duplicating an existing model under a new name."""
756 source: str
757 destination: str
760@router.post('/api/copy')
761@router.post('/api/copy/{url_idx}')
762async def copy_model(
763 request: Request,
764 form_data: CopyModelForm,
765 url_idx: int | None = None,
766 user=Depends(get_admin_user),
767):
768 """Duplicate an existing model under a new name."""
769 if not await Config.get('ollama.enable'):
770 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
772 if url_idx is None:
773 await get_all_models(request, user=user)
774 models = request.app.state.OLLAMA_MODELS
775 if form_data.source not in models: 775 ↛ 777line 775 didn't jump to line 777 because the condition on line 775 was always true
776 raise HTTPException(status_code=400, detail=ERROR_MESSAGES.MODEL_NOT_FOUND(form_data.source))
777 url_idx = models[form_data.source]['urls'][0]
779 url = (await Config.get('ollama.base_urls', []))[url_idx]
780 key = get_api_key(url_idx, url, (await Config.get('ollama.api_configs', {})))
782 await send_request(
783 f'{url}/api/copy',
784 payload=form_data.model_dump_json(exclude_none=True).encode(),
785 key=key,
786 user=user,
787 )
788 await publish_event(
789 request,
790 EVENTS.MODEL_PROVIDER_MODEL_CREATED,
791 actor=user,
792 subject_id=form_data.destination,
793 data={'provider': 'ollama', 'source': form_data.source, 'url_idx': url_idx},
794 )
795 return True
798@router.delete('/api/delete')
799@router.delete('/api/delete/{url_idx}')
800async def delete_model(
801 request: Request,
802 form_data: ModelNameForm,
803 url_idx: int | None = None,
804 user=Depends(get_admin_user),
805):
806 """Remove a model from an Ollama backend."""
807 if not await Config.get('ollama.enable'):
808 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
810 payload = form_data.model_dump(exclude_none=True)
811 payload['model'] = payload.get('model', payload.get('name'))
812 model = payload.get('model')
814 if url_idx is None:
815 await get_all_models(request, user=user)
816 models = request.app.state.OLLAMA_MODELS
817 if model not in models: 817 ↛ 819line 817 didn't jump to line 819 because the condition on line 817 was always true
818 raise HTTPException(status_code=400, detail=ERROR_MESSAGES.MODEL_NOT_FOUND(model))
819 url_idx = models[model]['urls'][0]
821 url = (await Config.get('ollama.base_urls', []))[url_idx]
822 key = get_api_key(url_idx, url, (await Config.get('ollama.api_configs', {})))
824 await send_request(
825 f'{url}/api/delete',
826 'DELETE',
827 payload=JSONCodec.dumps(payload),
828 key=key,
829 user=user,
830 )
831 await publish_event(
832 request,
833 EVENTS.MODEL_PROVIDER_MODEL_DELETED,
834 actor=user,
835 subject_id=model,
836 data={'provider': 'ollama', 'url_idx': url_idx},
837 )
838 return True
841@router.post('/api/show')
842async def show_model_info(
843 request: Request,
844 form_data: ModelNameForm,
845 user=Depends(get_verified_user),
846):
847 """Retrieve model metadata from the Ollama backend."""
848 if not await Config.get('ollama.enable'):
849 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
851 payload = form_data.model_dump(exclude_none=True)
852 payload['model'] = payload.get('model', payload.get('name'))
853 model = payload.get('model')
855 await check_model_access(user, await Models.get_model_by_id(model), BYPASS_MODEL_ACCESS_CONTROL)
857 await get_all_models(request, user=user)
858 models = request.app.state.OLLAMA_MODELS
860 if model not in models: 860 ↛ 863line 860 didn't jump to line 863 because the condition on line 860 was always true
861 raise HTTPException(status_code=400, detail=ERROR_MESSAGES.MODEL_NOT_FOUND(model))
863 url_idx = random.choice(models[model]['urls'])
864 url = (await Config.get('ollama.base_urls', []))[url_idx]
865 key = get_api_key(url_idx, url, (await Config.get('ollama.api_configs', {})))
867 return await send_request(
868 f'{url}/api/show',
869 payload=JSONCodec.dumps(payload),
870 key=key,
871 user=user,
872 )
875class GenerateEmbedForm(BaseModel):
876 """Payload for the newer /api/embed endpoint (batch-capable)."""
878 model: str
879 input: list[str] | str
880 truncate: bool | None = None
881 options: dict | None = None
882 keep_alive: Union[int, str | None] = None
883 model_config = ConfigDict(extra='allow')
886@router.post('/api/embed')
887@router.post('/api/embed/{url_idx}')
888async def embed(
889 request: Request,
890 form_data: GenerateEmbedForm,
891 url_idx: int | None = None,
892 user=Depends(get_verified_user),
893):
894 """Generate embeddings via the Ollama /api/embed endpoint."""
895 if not await Config.get('ollama.enable'):
896 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
898 log.info('generate_ollama_batch_embeddings %s', form_data)
899 await check_model_access(user, await Models.get_model_by_id(form_data.model), BYPASS_MODEL_ACCESS_CONTROL)
900 await validate_ollama_backend_idx(request, form_data.model, url_idx, user)
902 if url_idx is None:
903 model = form_data.model
904 models = request.app.state.OLLAMA_MODELS
905 if not models or model not in models: 905 ↛ 908line 905 didn't jump to line 908 because the condition on line 905 was always true
906 await get_all_models(request, user=user)
907 models = request.app.state.OLLAMA_MODELS
908 if model not in models: 908 ↛ 910line 908 didn't jump to line 910 because the condition on line 908 was always true
909 raise HTTPException(status_code=400, detail=ERROR_MESSAGES.MODEL_NOT_FOUND(form_data.model))
910 url_idx = random.choice(models[model]['urls'])
912 base_urls, api_configs = await get_ollama_connection_config()
913 url = base_urls[url_idx]
914 api_config = api_configs.get(str(url_idx), api_configs.get(url, {}))
915 key = get_api_key(url_idx, url, api_configs)
917 prefix_id = api_config.get('prefix_id')
918 form_data.model = strip_provider_model_prefix(form_data.model, prefix_id)
920 return await send_request(
921 f'{url}/api/embed',
922 payload=form_data.model_dump_json(exclude_none=True).encode(),
923 key=key,
924 user=user,
925 )
928class GenerateEmbeddingsForm(BaseModel):
929 """Payload for the legacy /api/embeddings endpoint (single-prompt)."""
931 model: str
932 prompt: str
933 options: dict | None = None
934 keep_alive: Union[int, str | None] = None
937@router.post('/api/embeddings')
938@router.post('/api/embeddings/{url_idx}')
939async def embeddings(
940 request: Request,
941 form_data: GenerateEmbeddingsForm,
942 url_idx: int | None = None,
943 user=Depends(get_verified_user),
944):
945 """Generate embeddings via the legacy Ollama /api/embeddings endpoint."""
946 if not await Config.get('ollama.enable'):
947 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
949 log.info('generate_ollama_embeddings %s', form_data)
950 await check_model_access(user, await Models.get_model_by_id(form_data.model), BYPASS_MODEL_ACCESS_CONTROL)
951 await validate_ollama_backend_idx(request, form_data.model, url_idx, user)
953 if url_idx is None:
954 model = form_data.model
955 models = request.app.state.OLLAMA_MODELS
956 if not models or model not in models: 956 ↛ 959line 956 didn't jump to line 959 because the condition on line 956 was always true
957 await get_all_models(request, user=user)
958 models = request.app.state.OLLAMA_MODELS
959 if model not in models: 959 ↛ 961line 959 didn't jump to line 961 because the condition on line 959 was always true
960 raise HTTPException(status_code=400, detail=ERROR_MESSAGES.MODEL_NOT_FOUND(form_data.model))
961 url_idx = random.choice(models[model]['urls'])
963 base_urls, api_configs = await get_ollama_connection_config()
964 url = base_urls[url_idx]
965 api_config = api_configs.get(str(url_idx), api_configs.get(url, {}))
966 key = get_api_key(url_idx, url, api_configs)
968 prefix_id = api_config.get('prefix_id')
969 form_data.model = strip_provider_model_prefix(form_data.model, prefix_id)
971 return await send_request(
972 f'{url}/api/embeddings',
973 payload=form_data.model_dump_json(exclude_none=True).encode(),
974 key=key,
975 user=user,
976 )
979class GenerateCompletionForm(BaseModel):
980 """Payload for the Ollama /api/generate endpoint."""
982 model: str
983 prompt: str | None = None
984 suffix: str | None = None
985 images: list[str] | None = None
986 format: Union[dict, str | None] = None
987 options: dict | None = None
988 system: str | None = None
989 template: str | None = None
990 context: list[int] | None = None
991 stream: bool | None = True
992 raw: bool | None = None
993 keep_alive: Union[int, str | None] = None
996@router.post('/api/generate')
997@router.post('/api/generate/{url_idx}')
998async def generate_completion(
999 request: Request,
1000 form_data: GenerateCompletionForm,
1001 url_idx: int | None = None,
1002 user=Depends(get_verified_user),
1003):
1004 """Run text completion via Ollama /api/generate."""
1005 if not await Config.get('ollama.enable'):
1006 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
1008 await check_model_access(user, await Models.get_model_by_id(form_data.model), BYPASS_MODEL_ACCESS_CONTROL)
1009 await validate_ollama_backend_idx(request, form_data.model, url_idx, user)
1011 if url_idx is None: 1011 ↛ 1019line 1011 didn't jump to line 1019 because the condition on line 1011 was always true
1012 await get_all_models(request, user=user)
1013 models = request.app.state.OLLAMA_MODELS
1014 model = form_data.model
1015 if model not in models: 1015 ↛ 1017line 1015 didn't jump to line 1017 because the condition on line 1015 was always true
1016 raise HTTPException(status_code=400, detail=ERROR_MESSAGES.MODEL_NOT_FOUND(form_data.model))
1017 url_idx = random.choice(models[model]['urls'])
1019 base_urls, api_configs = await get_ollama_connection_config()
1020 url = base_urls[url_idx]
1021 api_config = api_configs.get(str(url_idx), api_configs.get(url, {}))
1023 prefix_id = api_config.get('prefix_id')
1024 form_data.model = strip_provider_model_prefix(form_data.model, prefix_id)
1026 return await send_request(
1027 f'{url}/api/generate',
1028 payload=form_data.model_dump_json(exclude_none=True).encode(),
1029 key=get_api_key(url_idx, url, api_configs),
1030 user=user,
1031 stream=True,
1032 passthrough=True,
1033 )
1036class ChatMessage(BaseModel):
1037 """A single message in an Ollama chat conversation."""
1039 role: str
1040 content: str | None = None
1041 tool_calls: list[dict] | None = None
1042 images: list[str] | None = None
1043 model_config = ConfigDict(extra='allow')
1045 @validator('content', pre=True)
1046 @classmethod
1047 def check_at_least_one_field(cls, field_value, values, **kwargs):
1048 if field_value is None and ('tool_calls' not in values or values['tool_calls'] is None):
1049 raise ValueError("At least one of 'content' or 'tool_calls' must be provided")
1050 return field_value
1053class GenerateChatCompletionForm(BaseModel):
1054 """Payload for the Ollama /api/chat endpoint."""
1056 model: str
1057 messages: list[ChatMessage]
1058 format: Union[dict, str | None] = None
1059 options: dict | None = None
1060 template: str | None = None
1061 stream: bool | None = True
1062 keep_alive: Union[int, str | None] = None
1063 tools: list[dict] | None = None
1064 model_config = ConfigDict(extra='allow')
1067async def validate_ollama_backend_idx(request: Request, model: str, url_idx: int | None, user) -> None:
1068 # A caller-supplied url_idx must point to a backend the model is actually
1069 # served from; the None path is already constrained to that allow-list.
1070 if url_idx is None or user is None or getattr(user, 'role', None) == 'admin': 1070 ↛ 1072line 1070 didn't jump to line 1072 because the condition on line 1070 was always true
1071 return
1072 models = request.app.state.OLLAMA_MODELS
1073 if not models or model not in models:
1074 await get_all_models(request, user=user)
1075 models = request.app.state.OLLAMA_MODELS
1076 if url_idx not in (models.get(model) or {}).get('urls', []):
1077 raise HTTPException(status_code=403, detail=ERROR_MESSAGES.ACCESS_PROHIBITED)
1080async def get_ollama_url(request: Request, model: str, url_idx: int | None = None, user=None):
1081 await validate_ollama_backend_idx(request, model, url_idx, user)
1082 if url_idx is None: 1082 ↛ 1083line 1082 didn't jump to line 1083 because the condition on line 1082 was never true
1083 models = request.app.state.OLLAMA_MODELS
1084 if not models or model not in models:
1085 await get_all_models.cache.clear()
1086 await get_all_models(request, user=user)
1087 models = request.app.state.OLLAMA_MODELS
1088 if model not in models:
1089 raise HTTPException(
1090 status_code=400,
1091 detail=ERROR_MESSAGES.MODEL_NOT_FOUND(model),
1092 )
1093 url_idx = random.choice(models[model].get('urls', []))
1094 url = (await Config.get('ollama.base_urls', []))[url_idx]
1095 return url, url_idx
1098@router.post('/api/chat')
1099@router.post('/api/chat/{url_idx}')
1100async def generate_chat_completion(
1101 request: Request,
1102 form_data: dict,
1103 url_idx: int | None = None,
1104 user=Depends(get_verified_user), # noqa: B008
1105):
1106 """Forward a chat completion request to an Ollama backend."""
1107 if not await Config.get('ollama.enable'):
1108 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
1110 # NOTE: We intentionally do NOT use Depends(get_async_session) here.
1111 # Database operations (get_model_by_id, AccessGrants.has_access) manage their own short-lived sessions.
1112 # This prevents holding a connection during the entire LLM call (30-60+ seconds),
1113 # which would exhaust the connection pool under concurrent load.
1115 # bypass_filter and bypass_system_prompt are read from request.state to prevent
1116 # external clients from setting them via query parameter. Only internal
1117 # server-side callers (e.g. utils/chat.py) should set
1118 # request.state.bypass_filter / request.state.bypass_system_prompt = True.
1119 bypass_filter = getattr(request.state, 'bypass_filter', False)
1120 if BYPASS_MODEL_ACCESS_CONTROL: 1120 ↛ 1121line 1120 didn't jump to line 1121 because the condition on line 1120 was never true
1121 bypass_filter = True
1122 bypass_system_prompt = getattr(request.state, 'bypass_system_prompt', False)
1124 metadata = form_data.pop('metadata', None)
1125 try:
1126 form_data = GenerateChatCompletionForm(**form_data)
1127 except Exception as exc:
1128 log.exception(exc)
1129 raise HTTPException(status_code=400, detail=str(exc))
1131 if isinstance(form_data, BaseModel):
1132 payload = form_data.model_dump(exclude_none=True)
1134 payload.pop('metadata', None)
1136 model_id = payload['model']
1137 model_info = await Models.get_model_by_id(model_id)
1139 if model_info is not None:
1140 if model_info.base_model_id:
1141 base_model_id = request.base_model_id if hasattr(request, 'base_model_id') else model_info.base_model_id
1142 payload['model'] = base_model_id
1144 params = model_info.params.model_dump()
1145 if params:
1146 system = params.pop('system', None)
1147 payload = apply_model_params_to_body_ollama(params, payload)
1148 if not bypass_system_prompt:
1149 payload = await apply_system_prompt_to_body(system, payload, metadata, user)
1151 await check_model_access(user, model_info, bypass_filter)
1152 else:
1153 await check_model_access(user, None, bypass_filter)
1155 url, url_idx = await get_ollama_url(request, payload['model'], url_idx, user)
1156 api_configs = await Config.get('ollama.api_configs', {})
1157 api_config = resolve_api_config(api_configs, url_idx, url)
1159 prefix_id = api_config.get('prefix_id')
1160 payload['model'] = strip_provider_model_prefix(payload['model'], prefix_id)
1162 return await send_request(
1163 f'{url}/api/chat',
1164 payload=JSONCodec.dumps(payload),
1165 key=get_api_key(url_idx, url, api_configs),
1166 user=user,
1167 stream=form_data.stream,
1168 content_type='application/x-ndjson',
1169 metadata=metadata,
1170 api_config=api_config,
1171 request=request,
1172 )
1175# TODO: we should update this part once Ollama supports other types
1176class OpenAIChatMessageContent(BaseModel):
1177 """Content block within an OpenAI-style chat message."""
1179 type: str
1180 model_config = ConfigDict(extra='allow')
1183class OpenAIChatMessage(BaseModel):
1184 """A single message in an OpenAI-compatible chat request."""
1186 role: str
1187 content: Union[str | None, list[OpenAIChatMessageContent]]
1188 model_config = ConfigDict(extra='allow')
1191class OpenAIChatCompletionForm(BaseModel):
1192 """Payload for the OpenAI-compatible /v1/chat/completions proxy."""
1194 model: str
1195 messages: list[OpenAIChatMessage]
1196 model_config = ConfigDict(extra='allow')
1199class OpenAICompletionForm(BaseModel):
1200 """Payload for the OpenAI-compatible /v1/completions proxy."""
1202 model: str
1203 prompt: str
1204 model_config = ConfigDict(extra='allow')
1207class OpenAIEmbeddingsForm(BaseModel):
1208 """Payload for the OpenAI-compatible /v1/embeddings proxy."""
1210 model: str
1211 input: object
1212 model_config = ConfigDict(extra='allow')
1215@router.post('/v1/completions')
1216@router.post('/v1/completions/{url_idx}')
1217async def generate_openai_completion(
1218 request: Request,
1219 form_data: dict,
1220 url_idx: int | None = None,
1221 user=Depends(get_verified_user), # noqa: B008
1222):
1223 """Forward a text completion request via the OpenAI-compatible proxy."""
1224 # NOTE: We intentionally do NOT use Depends(get_async_session) here.
1225 # Database operations (get_model_by_id, AccessGrants.has_access) manage their own short-lived sessions.
1226 # This prevents holding a connection during the entire LLM call (30-60+ seconds),
1227 # which would exhaust the connection pool under concurrent load.
1228 metadata = form_data.pop('metadata', None)
1230 try:
1231 form_data = OpenAICompletionForm(**form_data)
1232 except Exception as exc:
1233 log.exception(exc)
1234 raise HTTPException(status_code=400, detail=str(exc))
1236 payload = form_data.model_dump(exclude_none=True, exclude=['metadata'])
1238 model_id = form_data.model
1239 model_info = await Models.get_model_by_id(model_id)
1240 if model_info is not None:
1241 if model_info.base_model_id:
1242 payload['model'] = model_info.base_model_id
1243 params = model_info.params.model_dump()
1244 if params:
1245 payload = apply_model_params_to_body_openai(params, payload)
1246 await check_model_access(user, model_info)
1247 else:
1248 await check_model_access(user, None)
1250 url, url_idx = await get_ollama_url(request, payload['model'], url_idx, user)
1251 api_configs = await Config.get('ollama.api_configs', {})
1252 api_config = resolve_api_config(api_configs, url_idx, url)
1254 prefix_id = api_config.get('prefix_id')
1255 payload['model'] = strip_provider_model_prefix(payload['model'], prefix_id)
1257 return await send_request(
1258 f'{url}/v1/completions',
1259 payload=JSONCodec.dumps(payload),
1260 key=get_api_key(url_idx, url, api_configs),
1261 user=user,
1262 stream=payload.get('stream', False),
1263 passthrough=True,
1264 metadata=metadata,
1265 api_config=api_config,
1266 request=request,
1267 )
1270@router.post('/v1/embeddings')
1271@router.post('/v1/embeddings/{url_idx}')
1272async def generate_openai_embeddings(
1273 request: Request,
1274 form_data: dict,
1275 url_idx: int | None = None,
1276 user=Depends(get_verified_user), # noqa: B008
1277):
1278 """Forward an embeddings request via the OpenAI-compatible proxy."""
1279 if not await Config.get('ollama.enable'): 1279 ↛ 1282line 1279 didn't jump to line 1282 because the condition on line 1279 was always true
1280 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
1282 metadata = form_data.pop('metadata', None)
1284 try:
1285 form_data = OpenAIEmbeddingsForm(**form_data)
1286 except Exception as exc:
1287 log.exception(exc)
1288 raise HTTPException(status_code=400, detail=str(exc))
1290 payload = form_data.model_dump(exclude_none=True)
1291 payload.pop('metadata', None)
1293 model_id = form_data.model
1294 model_info = await Models.get_model_by_id(model_id)
1295 if model_info is not None:
1296 if model_info.base_model_id:
1297 payload['model'] = model_info.base_model_id
1298 await check_model_access(user, model_info)
1299 else:
1300 await check_model_access(user, None)
1302 url, url_idx = await get_ollama_url(request, payload['model'], url_idx, user)
1303 api_config = resolve_api_config((await Config.get('ollama.api_configs', {})), url_idx, url)
1305 prefix_id = api_config.get('prefix_id')
1306 payload['model'] = strip_provider_model_prefix(payload['model'], prefix_id)
1308 return await send_request(
1309 f'{url}/v1/embeddings',
1310 payload=JSONCodec.dumps(payload),
1311 key=get_api_key(url_idx, url, (await Config.get('ollama.api_configs', {}))),
1312 user=user,
1313 metadata=metadata,
1314 api_config=api_config,
1315 request=request,
1316 )
1319@router.post('/v1/chat/completions')
1320@router.post('/v1/chat/completions/{url_idx}')
1321async def generate_openai_chat_completion(
1322 request: Request,
1323 form_data: dict,
1324 url_idx: int | None = None,
1325 user=Depends(get_verified_user), # noqa: B008
1326):
1327 """Forward a chat completion request via the OpenAI-compatible proxy."""
1328 # NOTE: We intentionally do NOT use Depends(get_async_session) here.
1329 # Database operations (get_model_by_id, AccessGrants.has_access) manage their own short-lived sessions.
1330 # This prevents holding a connection during the entire LLM call (30-60+ seconds),
1331 # which would exhaust the connection pool under concurrent load.
1332 metadata = form_data.pop('metadata', None)
1334 try:
1335 form_data = OpenAIChatCompletionForm(**form_data)
1336 except Exception as exc:
1337 log.exception(exc)
1338 raise HTTPException(status_code=400, detail=str(exc))
1340 payload = form_data.model_dump(exclude_none=True, exclude=['metadata'])
1342 model_id = form_data.model
1343 model_info = await Models.get_model_by_id(model_id)
1344 if model_info is not None:
1345 if model_info.base_model_id:
1346 payload['model'] = model_info.base_model_id
1348 params = model_info.params.model_dump()
1349 if params:
1350 system = params.pop('system', None)
1351 payload = apply_model_params_to_body_openai(params, payload)
1352 payload = await apply_system_prompt_to_body(system, payload, metadata, user)
1354 await check_model_access(user, model_info)
1355 else:
1356 await check_model_access(user, None)
1358 url, url_idx = await get_ollama_url(request, payload['model'], url_idx, user)
1359 api_configs = await Config.get('ollama.api_configs', {})
1360 api_config = resolve_api_config(api_configs, url_idx, url)
1362 prefix_id = api_config.get('prefix_id')
1363 payload['model'] = strip_provider_model_prefix(payload['model'], prefix_id)
1365 return await send_request(
1366 f'{url}/v1/chat/completions',
1367 payload=JSONCodec.dumps(payload),
1368 key=get_api_key(url_idx, url, api_configs),
1369 user=user,
1370 stream=payload.get('stream', False),
1371 passthrough=True,
1372 metadata=metadata,
1373 api_config=api_config,
1374 request=request,
1375 )
1378@router.post('/v1/messages')
1379@router.post('/v1/messages/{url_idx}')
1380async def generate_anthropic_messages(
1381 request: Request,
1382 form_data: dict,
1383 url_idx: int | None = None,
1384 user=Depends(get_verified_user),
1385):
1386 """
1387 Proxy for Ollama's Anthropic-compatible /v1/messages endpoint.
1389 Forwards the request as-is to the Ollama backend, applying the same
1390 model resolution, access control, and prefix_id handling used by
1391 the OpenAI-compatible /v1/chat/completions proxy.
1393 See https://docs.ollama.com/api/anthropic-compatibility
1394 """
1395 if not await Config.get('ollama.enable'): 1395 ↛ 1398line 1395 didn't jump to line 1398 because the condition on line 1395 was always true
1396 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
1398 payload = {**form_data}
1399 model_id = payload.get('model', '')
1401 model_info = await Models.get_model_by_id(model_id)
1402 if model_info:
1403 if model_info.base_model_id:
1404 payload['model'] = model_info.base_model_id
1406 await check_model_access(user, model_info)
1407 else:
1408 await check_model_access(user, None)
1410 url, url_idx = await get_ollama_url(request, payload['model'], url_idx, user)
1411 api_configs = await Config.get('ollama.api_configs', {})
1412 api_config = api_configs.get(str(url_idx), api_configs.get(url, {})) # Legacy support
1414 prefix_id = api_config.get('prefix_id', None)
1415 payload['model'] = strip_provider_model_prefix(payload['model'], prefix_id)
1417 return await send_request(
1418 f'{url}/v1/messages',
1419 payload=JSONCodec.dumps(payload),
1420 key=get_api_key(url_idx, url, api_configs),
1421 user=user,
1422 stream=payload.get('stream', False),
1423 passthrough=True,
1424 content_type='text/event-stream' if payload.get('stream', False) else None,
1425 api_config=api_config,
1426 request=request,
1427 )
1430class ResponsesForm(BaseModel):
1431 model: str
1433 model_config = ConfigDict(extra='allow')
1436@router.post('/v1/responses')
1437@router.post('/v1/responses/{url_idx}')
1438async def generate_responses(
1439 request: Request,
1440 form_data: ResponsesForm,
1441 url_idx: int | None = None,
1442 user=Depends(get_verified_user),
1443):
1444 """
1445 Proxy for Ollama's OpenAI-compatible /v1/responses endpoint.
1447 Forwards the request as-is to the Ollama backend, applying the same
1448 model resolution, access control, and prefix_id handling used by
1449 the OpenAI-compatible /v1/chat/completions proxy.
1451 See https://ollama.com/blog/responses-api
1452 """
1453 if not await Config.get('ollama.enable'):
1454 raise HTTPException(status_code=503, detail=ERROR_MESSAGES.OLLAMA_API_DISABLED)
1456 payload = form_data.model_dump()
1457 model_id = form_data.model
1459 model_info = await Models.get_model_by_id(model_id)
1460 if model_info: 1460 ↛ 1461line 1460 didn't jump to line 1461 because the condition on line 1460 was never true
1461 if model_info.base_model_id:
1462 payload['model'] = model_info.base_model_id
1464 await check_model_access(user, model_info)
1465 else:
1466 await check_model_access(user, None)
1468 url, url_idx = await get_ollama_url(request, payload['model'], url_idx, user)
1469 api_configs = await Config.get('ollama.api_configs', {})
1470 api_config = api_configs.get(str(url_idx), api_configs.get(url, {})) # Legacy support
1472 prefix_id = api_config.get('prefix_id', None)
1473 payload['model'] = strip_provider_model_prefix(payload['model'], prefix_id)
1475 return await send_request(
1476 f'{url}/v1/responses',
1477 payload=JSONCodec.dumps(payload),
1478 key=get_api_key(url_idx, url, api_configs),
1479 user=user,
1480 stream=payload.get('stream', False),
1481 passthrough=True,
1482 content_type='text/event-stream' if payload.get('stream', False) else None,
1483 api_config=api_config,
1484 request=request,
1485 )
1488@router.get('/v1/models')
1489@router.get('/v1/models/{url_idx}')
1490async def get_openai_models(
1491 request: Request,
1492 url_idx: int | None = None,
1493 user=Depends(get_verified_user),
1494 db: AsyncSession = Depends(get_async_session),
1495) -> dict:
1496 """List models in the OpenAI-compatible format."""
1497 if url_idx is not None and user.role != 'admin': 1497 ↛ 1498line 1497 didn't jump to line 1498 because the condition on line 1497 was never true
1498 raise HTTPException(status_code=401, detail=ERROR_MESSAGES.ACCESS_PROHIBITED)
1500 if url_idx is None:
1501 model_list = await get_all_models(request, user=user)
1502 raw_models = model_list['models']
1503 else:
1504 url = (await Config.get('ollama.base_urls', []))[url_idx]
1505 model_list = await send_request(f'{url}/api/tags', 'GET')
1506 raw_models = model_list.get('models', [])
1508 now_ts = int(time.time())
1509 models = [{'id': m['model'], 'object': 'model', 'created': now_ts, 'owned_by': 'openai'} for m in raw_models]
1511 if user.role == 'user' and not BYPASS_MODEL_ACCESS_CONTROL: 1511 ↛ 1512line 1511 didn't jump to line 1512 because the condition on line 1511 was never true
1512 model_ids = [m['id'] for m in models]
1513 model_infos = {mi.id: mi for mi in await Models.get_models_by_ids(model_ids, db=db)}
1514 user_group_ids = {g.id for g in await Groups.get_groups_by_member_id(user.id, db=db)}
1515 accessible_ids = await AccessGrants.get_accessible_resource_ids(
1516 user_id=user.id,
1517 resource_type='model',
1518 resource_ids=list(model_infos.keys()),
1519 permission='read',
1520 user_group_ids=user_group_ids,
1521 db=db,
1522 )
1523 models = [
1524 m for m in models if (mi := model_infos.get(m['id'])) and (user.id == mi.user_id or mi.id in accessible_ids)
1525 ]
1527 return {'data': models, 'object': 'list'}
1530class UrlForm(BaseModel):
1531 """Form carrying a single URL string."""
1533 url: str
1536class UploadBlobForm(BaseModel):
1537 """Form carrying a filename for blob uploads."""
1539 filename: str
1542def parse_huggingface_url(hf_url: str) -> str | None:
1543 """Extract the filename from a HuggingFace download URL."""
1544 try:
1545 return urlparse(hf_url).path.split('/')[-1]
1546 except (ValueError, IndexError):
1547 return None
1550async def download_file_stream(
1551 ollama_url: str,
1552 file_url: str,
1553 file_path: str,
1554 file_name: str,
1555 chunk_size: int = AIOHTTP_FILE_STREAM_CHUNK_SIZE,
1556):
1557 """Stream a model file download from *file_url*, then push the blob to Ollama."""
1558 current_size = os.path.getsize(file_path) if os.path.exists(file_path) else 0
1559 headers = {'Range': f'bytes={current_size}-'} if current_size > 0 else {}
1561 session = await get_session()
1562 async with session.get(
1563 file_url,
1564 headers=headers,
1565 ssl=AIOHTTP_CLIENT_SESSION_SSL,
1566 timeout=aiohttp.ClientTimeout(total=600),
1567 ) as response:
1568 total_size = int(response.headers.get('content-length', 0)) + current_size
1570 async with aiofiles.open(file_path, 'ab') as f:
1571 async for data in response.content.iter_chunked(chunk_size):
1572 current_size += len(data)
1573 await f.write(data)
1575 progress_total = total_size or current_size
1576 progress = round((current_size / progress_total) * 100, 2)
1577 yield f'data: {{"progress": {progress}, "completed": {current_size}, "total": {total_size}}}\n\n'
1579 done = True
1580 hashed = await asyncio.to_thread(calculate_sha256, file_path, chunk_size)
1582 blob_url = f'{ollama_url}/api/blobs/sha256:{hashed}'
1583 blob_size = await asyncio.to_thread(os.path.getsize, file_path)
1585 async def blob_chunks():
1586 async with aiofiles.open(file_path, 'rb') as blob_file:
1587 while chunk := await blob_file.read(chunk_size):
1588 yield chunk
1590 async with session.post(
1591 blob_url,
1592 data=blob_chunks(),
1593 headers={'Content-Length': str(blob_size)},
1594 ssl=AIOHTTP_CLIENT_SESSION_SSL,
1595 timeout=aiohttp.ClientTimeout(total=30),
1596 ) as blob_resp:
1597 if blob_resp.ok:
1598 await asyncio.to_thread(os.remove, file_path)
1599 yield f'data: {JSONCodec.dumps({"done": done, "blob": f"sha256:{hashed}", "name": file_name})}\n\n'
1600 else:
1601 raise RuntimeError('Ollama: Could not create blob, Please try again.')
1604@router.post('/models/download')
1605@router.post('/models/download/{url_idx}')
1606async def download_model(
1607 request: Request,
1608 form_data: UrlForm,
1609 url_idx: int | None = None,
1610 user=Depends(get_admin_user),
1611):
1612 """Download a GGUF model from HuggingFace or GitHub and register it with Ollama."""
1613 allowed_hosts = ['https://huggingface.co/', 'https://github.com/']
1614 if not any(form_data.url.startswith(host) for host in allowed_hosts): 1614 ↛ 1620line 1614 didn't jump to line 1620 because the condition on line 1614 was always true
1615 raise HTTPException(
1616 status_code=400,
1617 detail='Invalid file_url. Only URLs from allowed hosts are permitted.',
1618 )
1620 url = (await Config.get('ollama.base_urls', []))[url_idx if url_idx is not None else 0]
1621 file_name = parse_huggingface_url(form_data.url)
1623 if not file_name:
1624 return None
1626 file_path = os.path.join(UPLOAD_DIR, file_name)
1627 return StreamingResponse(
1628 download_file_stream(url, form_data.url, file_path, file_name),
1629 )
1632@router.post('/models/upload')
1633@router.post('/models/upload/{url_idx}')
1634async def upload_model(
1635 request: Request,
1636 file: UploadFile = File(...),
1637 url_idx: int | None = None,
1638 user=Depends(get_admin_user),
1639):
1640 """Upload a local model file, push it as a blob, and create the model in Ollama."""
1641 ollama_url = (await Config.get('ollama.base_urls', []))[url_idx if url_idx is not None else 0]
1643 filename = os.path.basename(file.filename)
1644 file_path = os.path.join(UPLOAD_DIR, filename)
1645 os.makedirs(UPLOAD_DIR, exist_ok=True)
1647 # Stage 1: persist the uploaded file to disk
1648 chunk_size = AIOHTTP_FILE_STREAM_CHUNK_SIZE
1650 async with aiofiles.open(file_path, 'wb') as out_f:
1651 while chunk := await file.read(chunk_size):
1652 await out_f.write(chunk)
1654 async def file_process_stream():
1655 nonlocal ollama_url
1656 total_size = os.path.getsize(file_path)
1657 log.info('Total Model Size: %s', total_size)
1659 # Stage 2: hash the file and emit SSE progress
1660 file_hash = await asyncio.to_thread(calculate_sha256, file_path, chunk_size)
1661 log.info('Model Hash: %s', file_hash)
1663 try:
1664 bytes_read = 0
1665 async with aiofiles.open(file_path, 'rb') as f:
1666 while chunk := await f.read(chunk_size):
1667 bytes_read += len(chunk)
1668 progress = round(bytes_read / total_size * 100, 2)
1669 event = JSONCodec.dumps({'progress': progress, 'total': total_size, 'completed': bytes_read})
1670 yield f'data: {event}\n\n'
1672 session = await get_session()
1673 blob_url = f'{ollama_url}/api/blobs/sha256:{file_hash}'
1675 async def blob_chunks():
1676 async with aiofiles.open(file_path, 'rb') as blob_file:
1677 while chunk := await blob_file.read(chunk_size):
1678 yield chunk
1680 async with session.post(
1681 blob_url,
1682 data=blob_chunks(),
1683 headers={'Content-Length': str(total_size)},
1684 ssl=AIOHTTP_CLIENT_SESSION_SSL,
1685 timeout=get_client_timeout(),
1686 ) as resp:
1687 if not resp.ok:
1688 raise Exception('Ollama: Could not create blob, Please try again.')
1690 log.info('Uploaded to /api/blobs')
1691 await asyncio.to_thread(os.remove, file_path)
1693 # Stage 4: create the model
1694 model, _ext = os.path.splitext(filename)
1695 log.info('Created Model: %s', model)
1697 create_payload = {
1698 'model': model,
1699 'files': {filename: f'sha256:{file_hash}'},
1700 }
1701 log.info('Model Payload: %s', create_payload)
1703 async with session.post(
1704 f'{ollama_url}/api/create',
1705 headers={'Content-Type': 'application/json'},
1706 data=JSONCodec.dumps(create_payload),
1707 ssl=AIOHTTP_CLIENT_SESSION_SSL,
1708 timeout=get_client_timeout(),
1709 ) as create_resp:
1710 if create_resp.ok:
1711 log.info('API SUCCESS!')
1712 event = JSONCodec.dumps(
1713 {'done': True, 'blob': f'sha256:{file_hash}', 'name': filename, 'model_created': model}
1714 )
1715 yield f'data: {event}\n\n'
1716 else:
1717 resp_text = await create_resp.text()
1718 raise Exception(f'Failed to create model in Ollama. {resp_text}')
1720 except Exception as exc:
1721 yield f'data: {JSONCodec.dumps({"error": str(exc)})}\n\n'
1723 return StreamingResponse(file_process_stream(), media_type='text/event-stream')