Coverage for api/utils/image_proxy/extension.py: 73%

1import asyncio 

2import mimetypes 

3from os.path import splitext 

4from urllib.parse import urlparse 

5 

6from django.conf import settings 

7 

8import aiohttp 

9import django_redis 

10import structlog 

11from asgiref.sync import sync_to_async 

12from redis.exceptions import ConnectionError 

13 

14from api.utils.aiohttp import get_aiohttp_session 

15from api.utils.image_proxy.dataclasses import MediaInfo 

16from api.utils.image_proxy.exception import UpstreamThumbnailException 

17 

18 

19logger = structlog.get_logger(__name__) 

20 

21 

22_HEAD_TIMEOUT = aiohttp.ClientTimeout(settings.THUMBNAIL_EXTENSION_REQUEST_TIMEOUT) 

23 

24 

25# Used to filter network errors during extension checks that we believe 

26# we should get errors (rather than warnings) for, e.g., a Sentry issue. 

27# As such, this should exclude errors where we don't think we have the 

28# ability to control the outcome, whether now or in the future. 

29# Some things, like SSL errors and timeouts, may be used to inform liveness 

30# checks or otherwise useful catalog information in the future. 

31# However, for now, we just need to avoid filling up our error backlog 

32# with things we aren't able to actually do anything about right now. 

33_NON_ACTIONABLE_NETWORK_EXCEPTIONS = ( 

34 aiohttp.ClientConnectorCertificateError, 

35 aiohttp.ClientConnectorSSLError, 

36 aiohttp.ClientConnectorError, 

37 aiohttp.ServerDisconnectedError, 

38 aiohttp.ServerTimeoutError, 

39 aiohttp.ClientOSError, 

40 asyncio.TimeoutError, 

41) 

42 

43 

44async def get_image_extension(media_info: MediaInfo) -> str | None: 

45 image_url = media_info.image_url 

46 

47 cache = django_redis.get_redis_connection("default") 

48 key = f"media:{media_info.media_identifier}:thumb_type" 

49 

50 ext = _get_file_extension_from_url(image_url) 

51 

52 if not ext: 

53 # If the extension is not present in the URL, try to get it from the redis cache 

54 try: 

55 ext = await sync_to_async(cache.get)(key) 

56 ext = ext.decode("utf-8") if ext else None 

57 except ConnectionError: 

58 logger.warning("Redis connect failed, cannot get cached image extension.") 

59 

60 if not ext: 

61 # If the extension is still not present, try getting it from the content type 

62 try: 

63 session = await get_aiohttp_session() 

64 

65 async with session.head( 

66 image_url, 

67 raise_for_status=True, 

68 timeout=_HEAD_TIMEOUT, 

69 trace_request_ctx={ 

70 "timing_event_name": "thumbnail_extension_request_timing", 

71 "timing_event_ctx": {"provider": media_info.media_provider}, 

72 }, 

73 ) as response: 

74 if response.headers and "Content-Type" in response.headers: 74 ↛ 78line 74 didn't jump to line 78 because the condition on line 74 was always true

75 content_type = response.headers["Content-Type"] 

76 ext = _get_file_extension_from_content_type(content_type) 

77 else: 

78 ext = None 

79 

80 await _cache_extension(cache, key, ext) 

81 except Exception as exc: 

82 # Aside from client errors, the timeout defined for `get_image_extension` 

83 # is generous, and if the head request exceeds it, we're comfortable saying 

84 # we'll skip generating this thumbnail. In the future, we might adjust 

85 # timeouts with per-provider granularity, but for now, we just have to 

86 # accept they will happen and are part of the set of non-actionable 

87 # networking errors that we don't need to report as errors to Sentry. 

88 if not isinstance(exc, asyncio.TimeoutError): 

89 if isinstance(exc, _NON_ACTIONABLE_NETWORK_EXCEPTIONS): 

90 log = logger.warning 

91 else: 

92 log = logger.error 

93 

94 log("upstream_thumbnail_exception", exc=exc, exc_info=True) 

95 

96 raise UpstreamThumbnailException( 

97 "Failed to render thumbnail due to inability to check media " 

98 f"type. {exc}" 

99 ) 

100 

101 return ext 

102 

103 

104@sync_to_async 

105def _cache_extension(cache, key, ext): 

106 try: 

107 cache.set(key, ext if ext else "unknown") 

108 except ConnectionError: 

109 logger.warning("Redis connect failed, cannot cache image extension.") 

110 

111 

112def _get_file_extension_from_url(image_url: str) -> str: 

113 """Return the image extension if present in the URL.""" 

114 parsed = urlparse(image_url) 

115 _, ext = splitext(parsed.path) 

116 return ext[1:].lower() # remove the leading dot 

117 

118 

119def _get_file_extension_from_content_type(content_type: str) -> str | None: 

120 """ 

121 Return the image extension if present in the Response's content type 

122 header. 

123 """ 

124 if ( 124 ↛ 130line 124 didn't jump to line 130 because the condition on line 124 was always true

125 content_type 

126 and "/" in content_type 

127 and (ext := mimetypes.guess_extension(content_type.split(";")[0], strict=False)) 

128 ): 

129 return ext.strip(".") 

130 return None