Coverage for chalicelib/core/errors/errors_details_ch.py: 50%
72 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-10 12:56 +0000
1import logging
3from chalicelib.core.errors import helper as errors_helper
4from chalicelib.utils import ch_client, exp_ch_helper
5from chalicelib.utils import helper
6from chalicelib.utils.TimeUTC import TimeUTC
7from chalicelib.utils.metrics_helper import get_step_size
8from uuid import uuid4
10logger = logging.getLogger(__name__)
13def __transform_map_to_tag(data, key1, key2, requested_key):
14 result = []
15 for i in data:
16 if requested_key == 0 and i.get(key1) is None and i.get(key2) is None:
17 result.append({"name": "all", "count": int(i.get("count"))})
18 elif requested_key == 1 and i.get(key1) is not None and i.get(key2) is None:
19 result.append({"name": i.get(key1), "count": int(i.get("count"))})
20 elif requested_key == 2 and i.get(key1) is not None and i.get(key2) is not None:
21 result.append({"name": i.get(key2), "count": int(i.get("count"))})
22 return result
25def __process_tags_map(row):
26 browsers_partition = row.pop("browsers_partition")
27 os_partition = row.pop("os_partition")
28 device_partition = row.pop("device_partition")
29 country_partition = row.pop("country_partition")
30 return [
31 {"name": "browser",
32 "partitions": __transform_map_to_tag(data=browsers_partition,
33 key1="browser",
34 key2="browser_version",
35 requested_key=1)},
36 {"name": "browser.ver",
37 "partitions": __transform_map_to_tag(data=browsers_partition,
38 key1="browser",
39 key2="browser_version",
40 requested_key=2)},
41 {"name": "OS",
42 "partitions": __transform_map_to_tag(data=os_partition,
43 key1="os",
44 key2="os_version",
45 requested_key=1)
46 },
47 {"name": "OS.ver",
48 "partitions": __transform_map_to_tag(data=os_partition,
49 key1="os",
50 key2="os_version",
51 requested_key=2)},
52 {"name": "device.family",
53 "partitions": __transform_map_to_tag(data=device_partition,
54 key1="device_type",
55 key2="device",
56 requested_key=1)},
57 {"name": "device",
58 "partitions": __transform_map_to_tag(data=device_partition,
59 key1="device_type",
60 key2="device",
61 requested_key=2)},
62 {"name": "country", "partitions": __transform_map_to_tag(data=country_partition,
63 key1="country",
64 key2="",
65 requested_key=1)}
66 ]
69def get_details(project_id, error_id, user_id, **data):
70 MAIN_SESSIONS_TABLE = exp_ch_helper.get_main_sessions_table(0)
71 MAIN_EVENTS_TABLE = exp_ch_helper.get_main_events_table(0)
73 ch_basic_query_errors = errors_helper.__get_basic_constraints_ch(time_constraint=False, table_name="errors",
74 type_condition=False)
75 ch_basic_query_sessions = ch_basic_query_errors[:]
76 ch_basic_query_errors.append("errors.`$event_name`='ERROR'")
77 ch_basic_query_errors.append("error_id = %(error_id)s")
79 with ch_client.ClickHouseClient() as ch:
80 data["startDate24"] = TimeUTC.now(-1)
81 data["endDate24"] = TimeUTC.now()
82 data["startDate30"] = TimeUTC.now(-30)
83 data["endDate30"] = TimeUTC.now()
85 density24 = int(data.get("density24", 24))
86 step_size24 = get_step_size(data["startDate24"], data["endDate24"], density24) * 1000
87 density30 = int(data.get("density30", 30))
88 step_size30 = get_step_size(data["startDate30"], data["endDate30"], density30) * 1000
89 params = {
90 "startDate24": data['startDate24'],
91 "endDate24": data['endDate24'],
92 "startDate30": data['startDate30'],
93 "endDate30": data['endDate30'],
94 "project_id": project_id,
95 "userId": user_id,
96 "step_size24": step_size24,
97 "step_size30": step_size30,
98 "error_id": error_id}
100 tmp_id = uuid4().hex
101 tmp_table1 = f"pre_processed_error_{tmp_id}_{TimeUTC.now()}"
102 tmp_table2 = f"pre_processed_{tmp_id}_{TimeUTC.now()}"
103 tmp_query1 = f"""CREATE TEMPORARY TABLE {tmp_table1} AS
104 SELECT error_id,
105 toString(`$properties`.name) AS name,
106 toString(`$properties`.message) AS message,
107 session_id,
108 created_at AS datetime
109 FROM {MAIN_EVENTS_TABLE} AS errors
110 WHERE {" AND ".join(ch_basic_query_errors)};"""
111 tmp_query2 = f"""CREATE TEMPORARY TABLE {tmp_table2} AS
112 SELECT error_id,
113 name,
114 message,
115 session_id,
116 errors.datetime AS datetime,
117 user_id,
118 user_browser,
119 user_browser_version,
120 user_os,
121 user_os_version,
122 user_device_type,
123 user_device,
124 user_country
125 FROM {MAIN_SESSIONS_TABLE} AS sessions INNER JOIN {tmp_table1} AS errors USING(session_id)
126 WHERE {" AND ".join(ch_basic_query_sessions)}
127 AND sessions.session_id IN (SELECT session_id FROM {tmp_table1});"""
128 main_ch_query = f"""\
129 SELECT %(error_id)s AS error_id, name, message,users,
130 first_occurrence,last_occurrence,last_session_id,
131 sessions,browsers_partition,os_partition,device_partition,
132 country_partition,chart24,chart30
133 FROM (SELECT error_id,
134 name,
135 message
136 FROM {tmp_table2}
137 LIMIT 1) AS details
138 INNER JOIN (SELECT COUNT(DISTINCT user_id) AS users,
139 COUNT(DISTINCT session_id) AS sessions
140 FROM {tmp_table2}
141 WHERE datetime >= toDateTime(%(startDate30)s / 1000)
142 AND datetime <= toDateTime(%(endDate30)s / 1000)
143 ) AS last_month_stats ON TRUE
144 INNER JOIN (SELECT toUnixTimestamp(max(datetime)) * 1000 AS last_occurrence,
145 toUnixTimestamp(min(datetime)) * 1000 AS first_occurrence
146 FROM {tmp_table2}) AS time_details ON TRUE
147 INNER JOIN (SELECT session_id AS last_session_id
148 FROM {tmp_table2}
149 ORDER BY datetime DESC
150 LIMIT 1) AS last_session_details ON TRUE
151 INNER JOIN (SELECT groupArray(details) AS browsers_partition
152 FROM (SELECT COUNT(1) AS count,
153 coalesce(nullIf(user_browser,''),toNullable('unknown')) AS browser,
154 coalesce(nullIf(user_browser_version,''),toNullable('unknown')) AS browser_version,
155 map('browser', browser,
156 'browser_version', browser_version,
157 'count', toString(count)) AS details
158 FROM {tmp_table2}
159 GROUP BY ROLLUP(browser, browser_version)
160 ORDER BY browser nulls first, browser_version nulls first, count DESC) AS mapped_browser_details
161 ) AS browser_details ON TRUE
162 INNER JOIN (SELECT groupArray(details) AS os_partition
163 FROM (SELECT COUNT(1) AS count,
164 coalesce(nullIf(user_os,''),toNullable('unknown')) AS os,
165 coalesce(nullIf(user_os_version,''),toNullable('unknown')) AS os_version,
166 map('os', os,
167 'os_version', os_version,
168 'count', toString(count)) AS details
169 FROM {tmp_table2}
170 GROUP BY ROLLUP(os, os_version)
171 ORDER BY os nulls first, os_version nulls first, count DESC) AS mapped_os_details
172 ) AS os_details ON TRUE
173 INNER JOIN (SELECT groupArray(details) AS device_partition
174 FROM (SELECT COUNT(1) AS count,
175 coalesce(nullIf(user_device,''),toNullable('unknown')) AS user_device,
176 map('device_type', toString(user_device_type),
177 'device', user_device,
178 'count', toString(count)) AS details
179 FROM {tmp_table2}
180 GROUP BY ROLLUP(user_device_type, user_device)
181 ORDER BY user_device_type nulls first, user_device nulls first, count DESC
182 ) AS count_per_device_details
183 ) AS mapped_device_details ON TRUE
184 INNER JOIN (SELECT groupArray(details) AS country_partition
185 FROM (SELECT COUNT(1) AS count,
186 map('country', toString(user_country),
187 'count', toString(count)) AS details
188 FROM {tmp_table2}
189 GROUP BY user_country
190 ORDER BY count DESC) AS count_per_country_details
191 ) AS mapped_country_details ON TRUE
192 INNER JOIN (SELECT groupArray(map('timestamp', timestamp, 'count', count)) AS chart24
193 FROM (SELECT gs.generate_series AS timestamp,
194 COUNT(DISTINCT session_id) AS count
195 FROM generate_series(%(startDate24)s, %(endDate24)s, %(step_size24)s) AS gs
196 LEFT JOIN {tmp_table2} ON(TRUE)
197 WHERE datetime >= toDateTime(timestamp / 1000)
198 AND datetime < toDateTime((timestamp + %(step_size24)s) / 1000)
199 GROUP BY timestamp
200 ORDER BY timestamp) AS chart_details
201 ) AS chart_details24 ON TRUE
202 INNER JOIN (SELECT groupArray(map('timestamp', timestamp, 'count', count)) AS chart30
203 FROM (SELECT gs.generate_series AS timestamp,
204 COUNT(DISTINCT session_id) AS count
205 FROM generate_series(%(startDate30)s, %(endDate30)s, %(step_size30)s) AS gs
206 LEFT JOIN {tmp_table2} ON(TRUE)
207 WHERE datetime >= toDateTime(timestamp / 1000)
208 AND datetime < toDateTime((timestamp + %(step_size30)s) / 1000)
209 GROUP BY timestamp
210 ORDER BY timestamp) AS chart_details
211 ) AS chart_details30 ON TRUE;"""
213 ch.execute(query=tmp_query1, parameters=params)
214 try:
215 ch.execute(query=tmp_query2, parameters=params)
216 row = ch.execute(query=main_ch_query, parameters=params)
217 finally:
218 ch.execute(query=f"DROP TEMPORARY TABLE IF EXISTS {tmp_table2};")
219 ch.execute(query=f"DROP TEMPORARY TABLE IF EXISTS {tmp_table1};")
220 if len(row) == 0: 220 ↛ 222line 220 didn't jump to line 222 because the condition on line 220 was always true
221 return {"errors": ["error not found"]}
222 row = row[0]
224 row["tags"] = __process_tags_map(row)
226 query = f"""SELECT session_id, toUnixTimestamp(datetime) * 1000 AS start_ts,
227 user_anonymous_id,user_id, user_uuid, user_browser, user_browser_version,
228 user_os, user_os_version, user_device, FALSE AS favorite, True AS viewed
229 FROM {MAIN_SESSIONS_TABLE} AS sessions
230 WHERE project_id = toUInt16(%(project_id)s)
231 AND session_id = %(session_id)s
232 ORDER BY datetime DESC
233 LIMIT 1;"""
234 params = {"project_id": project_id, "session_id": row["last_session_id"], "userId": user_id}
235 logger.debug("--------------------")
236 logging.debug(ch.format(query=query, parameters=params))
237 logger.debug("--------------------")
238 status = ch.execute(query=query, parameters=params)
240 if status is not None and len(status) > 0:
241 status = status[0]
242 row["favorite"] = status.pop("favorite")
243 row["viewed"] = status.pop("viewed")
244 row["last_hydrated_session"] = status
245 else:
246 row["last_hydrated_session"] = None
247 row["favorite"] = False
248 row["viewed"] = False
249 return {"data": helper.dict_to_camel_case(row)}