Coverage for api/utils/image_proxy/__init__.py: 78%
118 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-07 06:14 +0000
1from functools import wraps
2from typing import Literal
3from urllib.parse import urlparse
5from django.conf import settings
6from django.http import HttpResponse
7from rest_framework.exceptions import UnsupportedMediaType
9import aiohttp
10import django_redis
11import structlog
12from asgiref.sync import sync_to_async
13from redis.client import Redis
14from redis.exceptions import ConnectionError
16from api.utils.aiohttp import get_aiohttp_session
17from api.utils.image_proxy.dataclasses import MediaInfo, RequestConfig
18from api.utils.image_proxy.exception import UpstreamThumbnailException
19from api.utils.image_proxy.extension import get_image_extension
20from api.utils.image_proxy.photon import get_photon_request_params
21from api.utils.image_proxy.wikimedia import get_wikimedia_thumbnail_url
22from api.utils.tallies import get_monthly_timestamp
25logger = structlog.get_logger(__name__)
27HEADERS = {
28 "User-Agent": settings.OUTBOUND_USER_AGENT_TEMPLATE.format(
29 purpose="ThumbnailGeneration"
30 )
31}
33PHOTON_TYPES = {"gif", "jpg", "jpeg", "png", "webp"}
34ORIGINAL_TYPES = {"svg"}
36PHOTON = "photon"
37ORIGINAL = "original"
38THUMBNAIL_STRATEGY = Literal["photon_proxy", "original"]
41def get_request_params_for_extension(
42 ext: str,
43 headers: dict[str, str],
44 media_info: MediaInfo,
45 parsed_image_url: urlparse,
46 request_config: RequestConfig,
47) -> tuple[str, dict[str, str], dict[str, str]]:
48 """
49 Get the request params (url, params, headers) for the thumbnail proxy.
50 If the image type is supported by photon, we use photon, and compute the necessary
51 request params, if the file can be cached and returned as is (SVG), we do that,
52 otherwise we raise UnsupportedMediaType exception.
53 """
55 if media_info.media_provider == "wikimedia": 55 ↛ 60line 55 didn't jump to line 60 because the condition on line 55 was never true
56 # Wikimedia supports all file types with their own thumbnail
57 # endpoints; we still request the thumbnail through to Site Accelerator to offload repeated
58 # requests from Wikimedia's servers. Site Accelerator essentially acts as a cache,
59 # because we'll rely on Wikimedia to resize and compress the image as needed.
60 thumbnail_url = get_wikimedia_thumbnail_url(
61 media_info,
62 parsed_image_url,
63 request_config,
64 )
65 if thumbnail_url is not None:
66 return get_photon_request_params(
67 thumbnail_url,
68 # Because Wikimedia's thumbnails are configurable for compression
69 # and size, we can skip both operations in Site Accelerator
70 is_full_size=True,
71 is_compressed=False,
72 headers=headers,
73 )
74 # else, `get_wikimedia_thumbnail_url` was unable to create a thumbnail URL
75 # for this wikimedia image, so fallback to the default routine
77 if ext in PHOTON_TYPES: 77 ↛ 84line 77 didn't jump to line 84 because the condition on line 77 was always true
78 return get_photon_request_params(
79 parsed_image_url,
80 request_config.is_full_size,
81 request_config.is_compressed,
82 headers,
83 )
84 elif ext in ORIGINAL_TYPES:
85 return media_info.image_url, {}, headers
86 raise UnsupportedMediaType(
87 f"Image extension {ext} is not supported by the thumbnail proxy."
88 )
91@sync_to_async
92def _tally_response(
93 tallies_conn,
94 media_info: MediaInfo,
95 month: str,
96 domain: str,
97 status_code: int,
98):
99 """
100 Tally image proxy response.
102 Pulled into a separate function to help reduce overload when skimming
103 the `get` function, which is complex enough as is.
104 """
106 with tallies_conn.pipeline() as tallies:
107 tallies.incr(f"thumbnail_response_code:{month}:{status_code}")
108 tallies.incr(
109 f"thumbnail_response_code_by_domain:{domain}:{month}:{status_code}"
110 )
111 tallies.incr(
112 f"thumbnail_response_code_by_provider:{media_info.media_provider}:"
113 f"{month}:{status_code}"
114 )
115 try:
116 tallies.execute()
117 except ConnectionError:
118 logger.warning(
119 "Redis connect failed, thumbnail response codes not tallied."
120 )
123@sync_to_async
124def _tally_client_response_errors(tallies, month: str, domain: str, status: int):
125 try:
126 tallies.incr(f"thumbnail_http_error:{domain}:{month}:{status}")
127 except ConnectionError:
128 logger.warning("Redis connect failed, thumbnail HTTP errors not tallied.")
131# thmbfail == THuMBnail FAILures; this key path will exist for every thumbnail
132# requested, so it needs to be space efficient
133FAILURE_CACHE_KEY_TEMPLATE = "thmbfail:{ident}"
136def _cache_repeated_failures(_get):
137 """
138 Wrap ``image_proxy.get`` to cache repeated upstream failures
139 and avoid re-requesting images likely to fail.
141 Do this by incrementing a counter for each media identifier each time the inner request
142 fails. Before making thumbnail requests, check this counter. If it is above the configured
143 threshold, assume the request will fail again, and eagerly return a failed response without
144 sending the request upstream.
146 Additionally, if the request succeeds and the failure count is not 0, decrement the counter
147 to reflect the successful response, accounting for thumbnails that were temporarily flaky,
148 while still allowing them to get temporarily cached as a failure if additional requests fail
149 and push the counter over the threshold.
150 """
152 @wraps(_get)
153 async def do_cache(*args, **kwargs):
154 media_info: MediaInfo = args[0]
155 compressed_ident = str(media_info.media_identifier).replace("-", "")
156 redis_key = FAILURE_CACHE_KEY_TEMPLATE.format(ident=compressed_ident)
157 tallies: Redis = django_redis.get_redis_connection("tallies")
159 try:
160 cached_failure_count = await sync_to_async(tallies.get)(
161 redis_key,
162 )
163 cached_failure_count = (
164 int(cached_failure_count) if cached_failure_count is not None else 0
165 )
166 except ConnectionError:
167 # Ignore the connection error, treat it like it's never been cached
168 cached_failure_count = 0
170 if cached_failure_count > settings.THUMBNAIL_FAILURE_CACHE_TOLERANCE:
171 logger.info(
172 "%s thumbnail is too flaky, using cached failure response.",
173 media_info.media_identifier,
174 )
175 raise UpstreamThumbnailException("Thumbnail unavailable from provider.")
177 try:
178 response = await _get(*args, **kwargs)
179 if cached_failure_count > 0: 179 ↛ 184line 179 didn't jump to line 184 because the condition on line 179 was never true
180 # Decrement the key
181 # Do not delete it, because if it isn't 0, then it has failed before
182 # meaning we should continue to monitor it within the cache window
183 # in case the upstream is flaky and eventually goes over the tolerance
184 try:
185 await sync_to_async(tallies.decr)(redis_key)
186 await sync_to_async(tallies.expire)(
187 redis_key, settings.THUMBNAIL_FAILURE_CACHE_WINDOW_SECONDS
188 )
189 except ConnectionError:
190 logger.warning(
191 "Redis connect failed, thumbnail failure not decremented."
192 )
193 return response
194 except:
195 try:
196 await sync_to_async(tallies.incr)(redis_key)
197 # Call expire each time the key is incremented
198 # This pushes expiration out each time a new failure is cached
199 await sync_to_async(tallies.expire)(
200 redis_key, settings.THUMBNAIL_FAILURE_CACHE_WINDOW_SECONDS
201 )
202 except ConnectionError:
203 logger.warning(
204 "Redis connect failed, thumbnail failure not incremented."
205 )
206 raise
208 return do_cache
211_UPSTREAM_TIMEOUT = aiohttp.ClientTimeout(settings.THUMBNAIL_UPSTREAM_TIMEOUT)
214@_cache_repeated_failures
215async def get(
216 media_info: MediaInfo,
217 request_config: RequestConfig = RequestConfig(),
218) -> HttpResponse:
219 """
220 Retrieve the proxied image.
222 Proxy an image through Photon if its file type is supported, else return the
223 original image if the file type is SVG. Otherwise, raise an exception.
224 """
225 image_url = media_info.image_url
227 tallies = django_redis.get_redis_connection("tallies")
228 tallies_incr = sync_to_async(tallies.incr)
229 month = get_monthly_timestamp()
231 image_extension = await get_image_extension(media_info)
233 headers = {"Accept": request_config.accept_header} | HEADERS
235 parsed_image_url = urlparse(image_url)
236 domain = parsed_image_url.netloc
238 upstream_url, params, headers = get_request_params_for_extension(
239 image_extension,
240 headers,
241 media_info,
242 parsed_image_url,
243 request_config,
244 )
246 try:
247 session = await get_aiohttp_session()
249 async with session.get(
250 upstream_url,
251 timeout=_UPSTREAM_TIMEOUT,
252 params=params,
253 headers=headers,
254 trace_request_ctx={
255 "timing_event_name": "thumbnail_upstream_timing",
256 "timing_event_ctx": {
257 "provider": media_info.media_provider,
258 "image_url": media_info.image_url,
259 "image_extension": image_extension,
260 },
261 },
262 ) as upstream_response:
263 await _tally_response(
264 tallies, media_info, month, domain, upstream_response.status
265 )
267 upstream_response.raise_for_status()
269 status_code = upstream_response.status
270 content_type = upstream_response.headers.get("Content-Type")
272 content = await upstream_response.content.read()
274 return HttpResponse(
275 content,
276 status=status_code,
277 content_type=content_type,
278 )
279 except Exception as exc:
280 exception_name = f"{exc.__class__.__module__}.{exc.__class__.__name__}"
281 key = f"thumbnail_error:{exception_name}:{domain}:{month}"
283 try:
284 await tallies_incr(key)
285 except ConnectionError:
286 logger.warning("Redis connect failed, thumbnail errors not tallied.")
288 if isinstance(exc, aiohttp.ClientResponseError): 288 ↛ 299line 288 didn't jump to line 299 because the condition on line 288 was always true
289 status = exc.status
290 await _tally_client_response_errors(tallies, month, domain, status)
291 logger.warning(
292 "thumbnail_upstream_failure",
293 url=upstream_url,
294 status=status,
295 provider=media_info.media_provider,
296 exc=exc.message,
297 )
299 raise UpstreamThumbnailException(f"Failed to render thumbnail. {exc}")