Coverage for open_webui/routers/audio.py: 19%

1"""Audio router — TTS speech synthesis and STT transcription endpoints.""" 

2 

3import asyncio 

4import base64 

5import hashlib 

6import html 

7import io 

8import logging 

9import mimetypes 

10import os 

11import uuid 

12from fnmatch import fnmatch 

13from pathlib import Path 

14from typing import Optional 

15 

16import aiofiles 

17import aiohttp 

18from fastapi import ( 

19 APIRouter, 

20 Depends, 

21 File, 

22 Form, 

23 HTTPException, 

24 Request, 

25 UploadFile, 

26 status, 

27) 

28from fastapi.responses import FileResponse 

29from open_webui.config import ( 

30 CACHE_DIR, 

31 ELEVENLABS_API_BASE_URL, 

32 WHISPER_COMPUTE_TYPE, 

33 WHISPER_LANGUAGE, 

34 WHISPER_MODEL_AUTO_UPDATE, 

35 WHISPER_MODEL_DIR, 

36 WHISPER_MULTILINGUAL, 

37 WHISPER_VAD_FILTER, 

38) 

39from open_webui.constants import ERROR_MESSAGES 

40from open_webui.env import ( 

41 AIOHTTP_CLIENT_SESSION_SSL, 

42 AIOHTTP_CLIENT_TIMEOUT, 

43 AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST, 

44 AIOHTTP_FILE_STREAM_CHUNK_SIZE, 

45 BYPASS_PYDUB_PREPROCESSING, 

46 DEVICE_TYPE, 

47 ENABLE_FORWARD_USER_INFO_HEADERS, 

48 ENV, 

49 USE_SLIM, 

50) 

51from open_webui.events import EVENTS, publish_event 

52from open_webui.models.config import Config 

53from open_webui.utils.access_control import has_permission 

54from open_webui.utils.auth import get_admin_user, get_verified_user 

55from open_webui.utils.headers import include_user_info_headers 

56from open_webui.utils.json_codec import JSONCodec 

57from open_webui.utils.misc import strict_match_mime_type 

58from open_webui.utils.session_pool import get_session 

59from pydantic import BaseModel 

60 

61# pydub needs stdlib audioop (gone in 3.13); keep requires-python capped < 3.13 

62if not USE_SLIM: 62 ↛ 67line 62 didn't jump to line 67 because the condition on line 62 was always true

63 from pydub import AudioSegment 

64 from pydub.silence import split_on_silence 

65 from pydub.utils import mediainfo 

66 

67log = logging.getLogger(__name__) 

68router = APIRouter() 

69 

70# --- Constants --- 

71 

72MAX_FILE_SIZE_MB: int = 20 

73MAX_FILE_SIZE: int = MAX_FILE_SIZE_MB * 1024 * 1024 

74AZURE_MAX_FILE_SIZE_MB: int = 200 

75AZURE_MAX_FILE_SIZE: int = AZURE_MAX_FILE_SIZE_MB * 1024 * 1024 

76 

77SPEECH_CACHE_DIR = CACHE_DIR / 'audio' / 'speech' 

78SPEECH_CACHE_DIR.mkdir(parents=True, exist_ok=True) 

79 

80TTS_CONFIG_KEYS = { 

81 'OPENAI_API_BASE_URL': 'audio.tts.openai.api_base_url', 

82 'OPENAI_API_KEY': 'audio.tts.openai.api_key', 

83 'OPENAI_PARAMS': 'audio.tts.openai.params', 

84 'API_KEY': 'audio.tts.api_key', 

85 'ENGINE': 'audio.tts.engine', 

86 'MODEL': 'audio.tts.model', 

87 'VOICE': 'audio.tts.voice', 

88 'SPLIT_ON': 'audio.tts.split_on', 

89 'AZURE_SPEECH_REGION': 'audio.tts.azure.speech_region', 

90 'AZURE_SPEECH_BASE_URL': 'audio.tts.azure.speech_base_url', 

91 'AZURE_SPEECH_OUTPUT_FORMAT': 'audio.tts.azure.speech_output_format', 

92 'MISTRAL_API_KEY': 'audio.tts.mistral.api_key', 

93 'MISTRAL_API_BASE_URL': 'audio.tts.mistral.api_base_url', 

94} 

95 

96STT_CONFIG_KEYS = { 

97 'OPENAI_API_BASE_URL': 'audio.stt.openai.api_base_url', 

98 'OPENAI_API_KEY': 'audio.stt.openai.api_key', 

99 'OPENAI_API_REQUEST_FORMAT': 'audio.stt.openai.api_request_format', 

100 'ENGINE': 'audio.stt.engine', 

101 'MODEL': 'audio.stt.model', 

102 'SUPPORTED_CONTENT_TYPES': 'audio.stt.supported_content_types', 

103 'ALLOWED_EXTENSIONS': 'audio.stt.allowed_extensions', 

104 'WHISPER_MODEL': 'audio.stt.whisper_model', 

105 'DEEPGRAM_API_KEY': 'audio.stt.deepgram.api_key', 

106 'AZURE_API_KEY': 'audio.stt.azure.api_key', 

107 'AZURE_REGION': 'audio.stt.azure.region', 

108 'AZURE_LOCALES': 'audio.stt.azure.locales', 

109 'AZURE_BASE_URL': 'audio.stt.azure.base_url', 

110 'AZURE_MAX_SPEAKERS': 'audio.stt.azure.max_speakers', 

111 'MISTRAL_API_KEY': 'audio.stt.mistral.api_key', 

112 'MISTRAL_API_BASE_URL': 'audio.stt.mistral.api_base_url', 

113 'MISTRAL_USE_CHAT_COMPLETIONS': 'audio.stt.mistral.use_chat_completions', 

114} 

115 

116 

117async def get_config_values(key_map: dict[str, str]) -> dict: 

118 values = await Config.get_many(*key_map.values()) 

119 return {field: values[storage_key] for field, storage_key in key_map.items() if storage_key in values} 

120 

121 

122def config_updates(data: dict, key_map: dict[str, str]) -> dict: 

123 return {key_map[field]: value for field, value in data.items() if field in key_map} 

124 

125 

126def is_audio_conversion_required(file_path): 

127 """ 

128 Check if the given audio file needs conversion to mp3. 

129 """ 

130 SUPPORTED_FORMATS = {'flac', 'm4a', 'mp3', 'mp4', 'mpeg', 'wav', 'webm'} 

131 

132 if not os.path.isfile(file_path): 

133 log.error(f'File not found: {file_path}') 

134 return False 

135 

136 try: 

137 info = mediainfo(file_path) 

138 codec_name = info.get('codec_name', '').lower() 

139 codec_type = info.get('codec_type', '').lower() 

140 codec_tag_string = info.get('codec_tag_string', '').lower() 

141 

142 if codec_name == 'aac' and codec_type == 'audio' and codec_tag_string == 'mp4a': 

143 # File is AAC/mp4a audio, recommend mp3 conversion 

144 return True 

145 

146 # If the codec name is in the supported formats 

147 if codec_name in SUPPORTED_FORMATS: 

148 return False 

149 

150 return True 

151 except Exception as e: 

152 log.error(f'Error getting audio format: {e}') 

153 return False 

154 

155 

156def convert_audio_to_mp3(file_path): 

157 """Convert audio file to mp3 format.""" 

158 try: 

159 output_path = os.path.splitext(file_path)[0] + '.mp3' 

160 audio = AudioSegment.from_file(file_path) 

161 audio.export(output_path, format='mp3') 

162 log.info('Converted %s to %s', file_path, output_path) 

163 return output_path 

164 except Exception as e: 

165 log.error(f'Error converting audio file: {e}') 

166 return None 

167 

168 

169def transcode_audio_to_mp3(audio_data: bytes, content_type_header: str, output_path: str) -> bool: 

170 """ 

171 Transcode audio bytes to MP3 if the Content-Type indicates a non-MP3 format. 

172 

173 Handles raw PCM audio (e.g. Gemini-TTS via OpenRouter/LiteLLM) by parsing 

174 optional rate/channels from the Content-Type params, defaulting to 24kHz, 

175 16-bit, mono. For other non-MP3 formats, uses pydub auto-detection. 

176 

177 Returns True if transcoding was performed, False if the data is already MP3. 

178 Respects BYPASS_PYDUB_PREPROCESSING — when set, writes raw bytes and logs a warning. 

179 """ 

180 mime_type = content_type_header.split(';')[0].strip().lower() 

181 

182 if mime_type in ('audio/mpeg', 'audio/mp3'): 

183 return False 

184 

185 if BYPASS_PYDUB_PREPROCESSING: 

186 log.warning( 

187 f'TTS returned {mime_type} but BYPASS_PYDUB_PREPROCESSING is set; writing raw audio without transcoding' 

188 ) 

189 return False 

190 

191 if mime_type in ('audio/pcm', 'audio/l16', 'audio/raw'): 

192 # Parse optional rate/channels from Content-Type params, 

193 # default: 24kHz, 16-bit, mono (standard for Gemini TTS). 

194 ct_params = {} 

195 for part in content_type_header.split(';')[1:]: 

196 key_val = part.strip().split('=') 

197 if len(key_val) == 2: 

198 ct_params[key_val[0].strip().lower()] = key_val[1].strip() 

199 

200 sample_rate = int(ct_params.get('rate', 24000)) 

201 channels = int(ct_params.get('channels', 1)) 

202 

203 audio_segment = AudioSegment.from_raw( 

204 io.BytesIO(audio_data), 

205 sample_width=2, 

206 frame_rate=sample_rate, 

207 channels=channels, 

208 ) 

209 else: 

210 audio_segment = AudioSegment.from_file(io.BytesIO(audio_data)) 

211 

212 audio_segment.export(str(output_path), format='mp3') 

213 log.info('Transcoded %s audio to MP3: %s', mime_type, output_path) 

214 return True 

215 

216 

217def set_faster_whisper_model(model: str, auto_update: bool = False): 

218 if USE_SLIM: 218 ↛ 219line 218 didn't jump to line 219 because the condition on line 218 was never true

219 raise HTTPException(503, 'Configure an external speech-to-text engine. Local Whisper is unavailable in slim.') 

220 whisper_model = None 

221 if model: 

222 from faster_whisper import WhisperModel 

223 

224 faster_whisper_kwargs = { 

225 'model_size_or_path': model, 

226 'device': DEVICE_TYPE if DEVICE_TYPE and DEVICE_TYPE == 'cuda' else 'cpu', 

227 'compute_type': WHISPER_COMPUTE_TYPE, 

228 'download_root': WHISPER_MODEL_DIR, 

229 'local_files_only': not auto_update, 

230 } 

231 

232 try: 

233 whisper_model = WhisperModel(**faster_whisper_kwargs) 

234 except Exception: 

235 log.warning('WhisperModel initialization failed, attempting download with local_files_only=False') 

236 faster_whisper_kwargs['local_files_only'] = False 

237 whisper_model = WhisperModel(**faster_whisper_kwargs) 

238 return whisper_model 

239 

240 

241class TTSConfigForm(BaseModel): 

242 OPENAI_API_BASE_URL: str 

243 OPENAI_API_KEY: str 

244 OPENAI_PARAMS: Optional[dict] = None 

245 API_KEY: str 

246 ENGINE: str 

247 MODEL: str 

248 VOICE: str 

249 SPLIT_ON: str 

250 AZURE_SPEECH_REGION: str 

251 AZURE_SPEECH_BASE_URL: str 

252 AZURE_SPEECH_OUTPUT_FORMAT: str 

253 MISTRAL_API_KEY: str 

254 MISTRAL_API_BASE_URL: str 

255 

256 

257class STTConfigForm(BaseModel): 

258 OPENAI_API_BASE_URL: str 

259 OPENAI_API_KEY: str 

260 OPENAI_API_REQUEST_FORMAT: str = 'multipart' 

261 ENGINE: str 

262 MODEL: str 

263 SUPPORTED_CONTENT_TYPES: list[str] = [] 

264 ALLOWED_EXTENSIONS: list[str] = [] 

265 WHISPER_MODEL: str 

266 DEEPGRAM_API_KEY: str 

267 AZURE_API_KEY: str 

268 AZURE_REGION: str 

269 AZURE_LOCALES: str 

270 AZURE_BASE_URL: str 

271 AZURE_MAX_SPEAKERS: str 

272 MISTRAL_API_KEY: str 

273 MISTRAL_API_BASE_URL: str 

274 MISTRAL_USE_CHAT_COMPLETIONS: bool 

275 

276 

277class AudioConfigUpdateForm(BaseModel): 

278 tts: TTSConfigForm 

279 stt: STTConfigForm 

280 

281 

282@router.get('/config') 

283async def get_audio_config(request: Request, user=Depends(get_admin_user)): 

284 return { 

285 'tts': await get_config_values(TTS_CONFIG_KEYS), 

286 'stt': await get_config_values(STT_CONFIG_KEYS), 

287 } 

288 

289 

290@router.post('/config/update') 

291async def update_audio_config(request: Request, form_data: AudioConfigUpdateForm, user=Depends(get_admin_user)): 

292 if USE_SLIM: 292 ↛ 293line 292 didn't jump to line 293 because the condition on line 292 was never true

293 current = await Config.get_many('audio.stt.engine', 'audio.tts.engine') 

294 if form_data.stt.ENGINE == '' and current.get('audio.stt.engine') != '': 

295 raise HTTPException(400, 'Local Whisper is unavailable in slim. Select an external speech-to-text engine.') 

296 if form_data.tts.ENGINE == 'transformers' and current.get('audio.tts.engine') != 'transformers': 

297 raise HTTPException(400, 'Local TTS is unavailable in slim. Select an external text-to-speech engine.') 

298 await Config.upsert( 

299 { 

300 **config_updates(form_data.tts.model_dump(exclude_unset=True), TTS_CONFIG_KEYS), 

301 **config_updates(form_data.stt.model_dump(exclude_unset=True), STT_CONFIG_KEYS), 

302 } 

303 ) 

304 

305 if form_data.stt.ENGINE == '' and not USE_SLIM: 

306 request.app.state.faster_whisper_model = await asyncio.to_thread( 

307 set_faster_whisper_model, form_data.stt.WHISPER_MODEL, WHISPER_MODEL_AUTO_UPDATE 

308 ) 

309 else: 

310 request.app.state.faster_whisper_model = None 

311 

312 config = await get_audio_config(request, user) 

313 await publish_event( 

314 request, 

315 EVENTS.CONFIG_UPDATED, 

316 actor=user, 

317 subject_id='audio', 

318 data={ 

319 'tts_engine': config.get('tts', {}).get('ENGINE'), 

320 'stt_engine': config.get('stt', {}).get('ENGINE'), 

321 }, 

322 ) 

323 return config 

324 

325 

326def load_speech_pipeline(request): 

327 if USE_SLIM: 

328 raise HTTPException(503, 'Configure an external text-to-speech engine. Local TTS is unavailable in slim.') 

329 from datasets import load_dataset 

330 from transformers import pipeline 

331 

332 if request.app.state.speech_synthesiser is None: 

333 request.app.state.speech_synthesiser = pipeline('text-to-speech', 'microsoft/speecht5_tts') 

334 

335 if request.app.state.speech_speaker_embeddings_dataset is None: 

336 request.app.state.speech_speaker_embeddings_dataset = load_dataset( 

337 'Matthijs/cmu-arctic-xvectors', split='validation' 

338 ) 

339 

340 

341async def _raise_tts_error(exc: Exception, r=None) -> None: 

342 """Raise a standardised HTTPException from a TTS provider failure.""" 

343 if isinstance(exc, HTTPException): 

344 raise exc 

345 code = r.status if r is not None and r.status >= 400 else 500 

346 # LICENSE covers this Open WebUI error identifier. 

347 # Do not alter, remove, obscure, or replace it except as LICENSE permits: 

348 # https://docs.openwebui.com/license. 

349 detail = 'Open WebUI: Server Connection Error' 

350 if r is not None: 

351 try: 

352 res = await r.json() 

353 if 'error' in res: 

354 msg = res['error'] 

355 detail = f'External: {msg.get("message", msg) if isinstance(msg, dict) else msg}' 

356 elif 'message' in res: 

357 detail = f'External: {res["message"]}' 

358 except Exception: 

359 detail = f'External: {exc}' 

360 raise HTTPException(status_code=code, detail=detail) 

361 

362 

363async def _write_tts_cache( 

364 file_path: Path, 

365 audio: bytes, 

366 body_path: Path, 

367 payload: dict, 

368 content_type: str = 'audio/mpeg', 

369) -> None: 

370 """Persist audio + request metadata to the speech cache.""" 

371 if USE_SLIM: 

372 mime_type = content_type.split(';')[0].strip().lower() 

373 if mime_type not in { 

374 'audio/mpeg', 

375 'audio/mp3', 

376 'audio/wav', 

377 'audio/x-wav', 

378 'audio/ogg', 

379 'audio/opus', 

380 'audio/webm', 

381 'audio/flac', 

382 'audio/aac', 

383 'audio/mp4', 

384 }: 

385 raise HTTPException( 

386 502, 

387 f'TTS returned unsupported format {mime_type}. Configure the provider to return MP3, WAV, Ogg, or another browser-playable audio format.', 

388 ) 

389 async with aiofiles.open(file_path.with_suffix('.mime'), 'w') as f: 

390 await f.write(content_type) 

391 async with aiofiles.open(file_path, 'wb') as f: 

392 await f.write(audio) 

393 async with aiofiles.open(body_path, 'w') as f: 

394 await f.write(JSONCodec.dumps(payload)) 

395 

396 

397async def _tts_openai(request, payload, file_path, file_body_path, user): 

398 """Generate speech via an OpenAI-compatible TTS endpoint.""" 

399 payload['model'] = await Config.get('audio.tts.model') 

400 if not payload.get('voice'): 

401 payload['voice'] = await Config.get('audio.tts.voice') 

402 payload = {**payload, **(await Config.get('audio.tts.openai.params') or {})} 

403 api_key = await Config.get('audio.tts.openai.api_key') 

404 api_base_url = await Config.get('audio.tts.openai.api_base_url') 

405 

406 headers = { 

407 'Content-Type': 'application/json', 

408 'Authorization': f'Bearer {api_key}', 

409 } 

410 if ENABLE_FORWARD_USER_INFO_HEADERS: 

411 headers = include_user_info_headers(headers, user) 

412 

413 r = None 

414 try: 

415 session = await get_session() 

416 r = await session.post( 

417 url=f'{api_base_url}/audio/speech', 

418 json=payload, 

419 headers=headers, 

420 ssl=AIOHTTP_CLIENT_SESSION_SSL, 

421 ) 

422 r.raise_for_status() 

423 

424 audio_data = await r.read() 

425 content_type = r.headers.get('Content-Type', 'audio/mpeg') 

426 

427 if USE_SLIM: 

428 await _write_tts_cache(file_path, audio_data, file_body_path, payload, content_type) 

429 return FileResponse(file_path, media_type=content_type) 

430 

431 if not await asyncio.to_thread(transcode_audio_to_mp3, audio_data, content_type, file_path): 

432 async with aiofiles.open(file_path, 'wb') as f: 

433 await f.write(audio_data) 

434 

435 async with aiofiles.open(file_body_path, 'w') as f: 

436 await f.write(JSONCodec.dumps(payload)) 

437 

438 return FileResponse(file_path) 

439 except Exception as exc: 

440 log.exception(exc) 

441 await _raise_tts_error(exc, r) 

442 

443 

444async def _tts_elevenlabs(request, payload, file_path, file_body_path, user): 

445 """Generate speech via the ElevenLabs TTS API.""" 

446 voice_id = (payload.get('voice') or '').strip() 

447 if not voice_id: 

448 raise HTTPException(status_code=400, detail='Invalid voice id') 

449 

450 available_voices = await get_available_voices(request) 

451 if available_voices and voice_id not in available_voices: 

452 raise HTTPException(status_code=400, detail='Invalid voice id') 

453 

454 r = None 

455 try: 

456 session = await get_session() 

457 async with session.post( 

458 f'{ELEVENLABS_API_BASE_URL}/v1/text-to-speech/{voice_id}', 

459 json={ 

460 'text': payload['input'], 

461 'model_id': await Config.get('audio.tts.model'), 

462 'voice_settings': {'stability': 0.5, 'similarity_boost': 0.5}, 

463 }, 

464 headers={ 

465 'Accept': 'audio/mpeg', 

466 'Content-Type': 'application/json', 

467 'xi-api-key': await Config.get('audio.tts.api_key'), 

468 }, 

469 ssl=AIOHTTP_CLIENT_SESSION_SSL, 

470 ) as r: 

471 r.raise_for_status() 

472 content_type = r.headers.get('Content-Type', 'audio/mpeg') 

473 await _write_tts_cache(file_path, await r.read(), file_body_path, payload, content_type) 

474 return FileResponse(file_path, media_type=content_type if USE_SLIM else None) 

475 except Exception as exc: 

476 log.exception(exc) 

477 await _raise_tts_error(exc, r) 

478 

479 

480async def _tts_azure(request, payload, file_path, file_body_path, user): 

481 """Generate speech via Azure Cognitive Services TTS.""" 

482 az_region = await Config.get('audio.tts.azure.speech_region') or 'eastus' 

483 az_base = await Config.get('audio.tts.azure.speech_base_url') 

484 language = payload.get('voice') or await Config.get('audio.tts.voice') 

485 locale = '-'.join(language.split('-')[:2]) 

486 output_format = await Config.get('audio.tts.azure.speech_output_format') 

487 

488 ssml = ( 

489 f'<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xml:lang="{html.escape(locale)}">' 

490 f'<voice name="{html.escape(language)}">{html.escape(payload["input"])}</voice>' 

491 f'</speak>' 

492 ) 

493 

494 r = None 

495 try: 

496 session = await get_session() 

497 async with session.post( 

498 (az_base or f'https://{az_region}.tts.speech.microsoft.com') + '/cognitiveservices/v1', 

499 headers={ 

500 'Ocp-Apim-Subscription-Key': await Config.get('audio.tts.api_key'), 

501 'Content-Type': 'application/ssml+xml', 

502 'X-Microsoft-OutputFormat': output_format, 

503 }, 

504 data=ssml, 

505 ssl=AIOHTTP_CLIENT_SESSION_SSL, 

506 ) as r: 

507 r.raise_for_status() 

508 content_type = r.headers.get('Content-Type', 'audio/mpeg') 

509 await _write_tts_cache(file_path, await r.read(), file_body_path, payload, content_type) 

510 return FileResponse(file_path, media_type=content_type if USE_SLIM else None) 

511 except Exception as exc: 

512 log.exception(exc) 

513 await _raise_tts_error(exc, r) 

514 

515 

516async def _tts_transformers(request, payload, file_path, file_body_path, user): 

517 """Generate speech via the local HuggingFace SpeechT5 pipeline (thread-offloaded).""" 

518 if USE_SLIM: 

519 raise HTTPException(503, 'Configure an external text-to-speech engine. Local TTS is unavailable in slim.') 

520 import soundfile as sf 

521 import torch 

522 

523 await asyncio.to_thread(load_speech_pipeline, request) 

524 

525 embeddings = request.app.state.speech_speaker_embeddings_dataset 

526 model_name = await Config.get('audio.tts.model') 

527 

528 idx = 6799 

529 try: 

530 idx = embeddings['filename'].index(model_name) 

531 except (ValueError, KeyError): 

532 log.debug('Speaker embedding not found for %s, using default index %s', model_name, idx) 

533 

534 def _run_pipeline(): 

535 speaker_embedding = torch.tensor(embeddings[idx]['xvector']).unsqueeze(0) 

536 wav = request.app.state.speech_synthesiser( 

537 payload['input'], # raw text to synthesize 

538 forward_params={ 

539 'speaker_embeddings': speaker_embedding, 

540 }, 

541 ) 

542 sf.write(str(file_path), wav['audio'], samplerate=wav['sampling_rate']) 

543 

544 await asyncio.to_thread(_run_pipeline) 

545 

546 # Audio file already written by sf.write; just persist the request metadata. 

547 async with aiofiles.open(file_body_path, 'w') as f: 

548 await f.write(JSONCodec.dumps(payload)) 

549 return FileResponse(file_path) 

550 

551 

552async def _tts_mistral(request, payload, file_path, file_body_path, user): 

553 """Generate speech via the Mistral TTS API.""" 

554 api_key = await Config.get('audio.tts.mistral.api_key') 

555 api_base_url = await Config.get('audio.tts.mistral.api_base_url') or 'https://api.mistral.ai/v1' 

556 

557 if not api_key: 

558 raise HTTPException(status_code=400, detail='Mistral API key is required for Mistral TTS') 

559 

560 r = None 

561 try: 

562 session = await get_session() 

563 r = await session.post( 

564 url=f'{api_base_url}/audio/speech', 

565 json={ 

566 'input': payload.get('input', ''), # text to synthesize 

567 'model': await Config.get('audio.tts.model') or 'voxtral-mini-tts-2603', 

568 'voice_id': payload.get('voice', ''), 

569 'response_format': 'mp3', 

570 }, 

571 headers={ 

572 'Content-Type': 'application/json', 

573 'Authorization': f'Bearer {api_key}', 

574 }, 

575 ssl=AIOHTTP_CLIENT_SESSION_SSL, 

576 ) 

577 r.raise_for_status() 

578 

579 res = await r.json() 

580 audio_b64 = res.get('audio_data', '') 

581 if not audio_b64: 

582 raise ValueError('No audio_data in Mistral TTS response') 

583 

584 await _write_tts_cache(file_path, base64.b64decode(audio_b64), file_body_path, payload) 

585 return FileResponse(file_path) 

586 except Exception as exc: 

587 log.exception(exc) 

588 await _raise_tts_error(exc, r) 

589 

590 

591# Dispatcher map: engine name -> handler 

592_TTS_ENGINES = { 

593 'openai': _tts_openai, 

594 'elevenlabs': _tts_elevenlabs, 

595 'azure': _tts_azure, 

596 'transformers': _tts_transformers, 

597 'mistral': _tts_mistral, 

598} 

599 

600 

601@router.post('/speech') 

602async def speech(request: Request, user=Depends(get_verified_user)): 

603 engine = await Config.get('audio.tts.engine') 

604 if USE_SLIM and engine in ('', 'transformers'): 604 ↛ 605line 604 didn't jump to line 605 because the condition on line 604 was never true

605 raise HTTPException(503, 'Configure an external text-to-speech engine.') 

606 if engine == '': 

607 raise HTTPException( 

608 status_code=status.HTTP_404_NOT_FOUND, 

609 detail=ERROR_MESSAGES.NOT_FOUND, 

610 ) 

611 

612 if user.role != 'admin' and not await has_permission(user.id, 'chat.tts', await Config.get('user.permissions')): 612 ↛ 613line 612 didn't jump to line 613 because the condition on line 612 was never true

613 raise HTTPException( 

614 status_code=status.HTTP_403_FORBIDDEN, 

615 detail=ERROR_MESSAGES.ACCESS_PROHIBITED, 

616 ) 

617 

618 body = await request.body() 

619 name = hashlib.sha256( 

620 body 

621 + str(engine).encode('utf-8') 

622 + str(await Config.get('audio.tts.model')).encode('utf-8') 

623 + (b':slim' if USE_SLIM else b'') 

624 ).hexdigest() 

625 

626 file_path = SPEECH_CACHE_DIR.joinpath(f'{name}.mp3') 

627 file_body_path = SPEECH_CACHE_DIR.joinpath(f'{name}.json') 

628 

629 # Return cached result if available 

630 if file_path.is_file(): 630 ↛ 631line 630 didn't jump to line 631 because the condition on line 630 was never true

631 await publish_event( 

632 request, 

633 EVENTS.AUDIO_SPEECH_REQUESTED, 

634 actor=user, 

635 subject_id=name, 

636 data={'engine': engine, 'cached': True}, 

637 ) 

638 content_type = None 

639 if USE_SLIM: 

640 async with aiofiles.open(file_path.with_suffix('.mime')) as f: 

641 content_type = await f.read() 

642 return FileResponse(file_path, media_type=content_type) 

643 

644 try: 

645 payload = JSONCodec.loads(body) 

646 except Exception as exc: 

647 log.exception(exc) 

648 raise HTTPException(status_code=400, detail='Invalid JSON payload') 

649 

650 handler = _TTS_ENGINES.get(engine) 

651 if handler is None: 

652 raise HTTPException(status_code=400, detail=f'Unsupported TTS engine: {engine}') 

653 

654 response = await handler(request, payload, file_path, file_body_path, user) 

655 await publish_event( 

656 request, 

657 EVENTS.AUDIO_SPEECH_REQUESTED, 

658 actor=user, 

659 subject_id=name, 

660 data={ 

661 'engine': engine, 

662 'model': payload.get('model'), 

663 'input_preview': str(payload.get('input', ''))[:300], 

664 'cached': False, 

665 }, 

666 ) 

667 return response 

668 

669 

670async def _transcribe_whisper(request, file_path, languages, file_dir, id): 

671 if request.app.state.faster_whisper_model is None: 

672 request.app.state.faster_whisper_model = await asyncio.to_thread( 

673 set_faster_whisper_model, await Config.get('audio.stt.whisper_model') 

674 ) 

675 

676 model = request.app.state.faster_whisper_model 

677 

678 def _run(): 

679 segments, info = model.transcribe( 

680 file_path, 

681 beam_size=5, 

682 vad_filter=WHISPER_VAD_FILTER, 

683 language=languages[0], 

684 multilingual=WHISPER_MULTILINGUAL, 

685 ) 

686 log.info("Detected language '%s' with probability %f", info.language, info.language_probability) 

687 return ''.join([segment.text for segment in list(segments)]) 

688 

689 transcript = await asyncio.to_thread(_run) 

690 data = {'text': transcript.strip()} 

691 

692 async with aiofiles.open(os.path.join(file_dir, f'{id}.json'), 'w') as f: 

693 await f.write(JSONCodec.dumps(data)) 

694 

695 log.debug(data) 

696 return data 

697 

698 

699async def _transcribe_openai(request, file_path, filename, languages, file_dir, id, user=None): 

700 """Transcribe audio via an OpenAI-compatible STT endpoint.""" 

701 r = None 

702 try: 

703 session = await get_session() 

704 api_key = await Config.get('audio.stt.openai.api_key') 

705 api_base_url = await Config.get('audio.stt.openai.api_base_url') 

706 request_format = (await Config.get('audio.stt.openai.api_request_format') or 'multipart').lower() 

707 

708 headers = {'Authorization': f'Bearer {api_key}'} 

709 if user and ENABLE_FORWARD_USER_INFO_HEADERS: 

710 headers = include_user_info_headers(headers, user) 

711 

712 for language in languages: 

713 payload = {'model': await Config.get('audio.stt.model')} 

714 if language: 

715 payload['language'] = language 

716 

717 if request_format == 'json': 

718 ext = os.path.splitext(filename)[1].lower().lstrip('.') or 'wav' 

719 async with aiofiles.open(file_path, 'rb') as f: 

720 payload['input_audio'] = { 

721 'data': base64.b64encode(await f.read()).decode('utf-8'), 

722 'format': 'ogg' if ext == 'oga' else ext, 

723 } 

724 

725 r = await session.post( 

726 url=f'{api_base_url}/audio/transcriptions', 

727 headers={**headers, 'Content-Type': 'application/json'}, 

728 json=payload, 

729 ssl=AIOHTTP_CLIENT_SESSION_SSL, 

730 ) 

731 else: 

732 form_data = aiohttp.FormData() 

733 for key, value in payload.items(): 

734 form_data.add_field(key, str(value)) 

735 

736 async def audio_chunks(): 

737 async with aiofiles.open(file_path, 'rb') as audio_file: 

738 while chunk := await audio_file.read(AIOHTTP_FILE_STREAM_CHUNK_SIZE): 

739 yield chunk 

740 

741 form_data.add_field('file', audio_chunks(), filename=filename) 

742 

743 r = await session.post( 

744 url=f'{api_base_url}/audio/transcriptions', 

745 headers=headers, 

746 data=form_data, 

747 ssl=AIOHTTP_CLIENT_SESSION_SSL, 

748 ) 

749 if r.status == 200: 

750 break 

751 

752 r.raise_for_status() 

753 data = await r.json() 

754 

755 async with aiofiles.open(os.path.join(file_dir, f'{id}.json'), 'w') as f: 

756 await f.write(JSONCodec.dumps(data)) 

757 return data 

758 except Exception as e: 

759 log.exception(e) 

760 detail = None 

761 if r is not None: 

762 try: 

763 res = await r.json() 

764 if 'error' in res: 

765 detail = f'External: {res["error"].get("message", "")}' 

766 except Exception: 

767 detail = f'External: {e}' 

768 # LICENSE covers this Open WebUI error identifier. 

769 # Do not alter, remove, obscure, or replace it except as LICENSE permits: 

770 # https://docs.openwebui.com/license. 

771 raise Exception(detail if detail else 'Open WebUI: Server Connection Error') 

772 

773 

774async def _transcribe_deepgram(request, file_path, languages, file_dir, id): 

775 """Transcribe audio via the Deepgram listen API with language fallback.""" 

776 content_type = mimetypes.guess_type(file_path)[0] or 'audio/wav' 

777 

778 async with aiofiles.open(file_path, 'rb') as f: 

779 audio_bytes = await f.read() 

780 

781 api_key = await Config.get('audio.stt.deepgram.api_key') 

782 stt_model = await Config.get('audio.stt.model') 

783 

784 r = None 

785 try: 

786 session = await get_session() 

787 for lang in languages: 

788 query: dict = {'smart_format': 'true'} 

789 if stt_model: