Coverage for conf/settings/logging.py: 66%

1import logging 

2from logging import LogRecord 

3 

4import structlog 

5from decouple import config 

6from structlog_sentry import SentryProcessor 

7 

8from conf.settings.base import ENVIRONMENT, INSTALLED_APPS, MIDDLEWARE 

9from conf.settings.security import DEBUG 

10 

11 

12def health_check_filter(record: LogRecord) -> bool: 

13 # Filter out health checks from the logs, they're verbose and happen frequently 

14 return not ("GET /healthcheck" in record.getMessage() and record.status_code == 200) 

15 

16 

17# https://django-structlog.readthedocs.io/en/latest/getting_started.html#installation 

18if "django_structlog" not in INSTALLED_APPS: 18 ↛ 21line 18 didn't jump to line 21 because the condition on line 18 was always true

19 INSTALLED_APPS.append("django_structlog") 

20 

21MIDDLEWARE.insert(0, "django_structlog.middlewares.RequestMiddleware") 

22 

23DJANGO_LOG_LEVEL = config("DJANGO_LOG_LEVEL", default="INFO").upper() 

24DJANGO_DATABASE_LOGGING = config("DJANGO_DATABASE_LOGGING", cast=bool, default=False) 

25LOG_PROCESSOR = config( 

26 "LOG_PROCESSOR", 

27 default="console" if ENVIRONMENT == "local" else "json", 

28) 

29 

30# Set to a pipe-delimited string of gc debugging flags 

31# https://docs.python.org/3/library/gc.html#gc.DEBUG_STATS 

32GC_DEBUG_LOGGING = config( 

33 "GC_DEBUG_LOGGING", cast=lambda x: x.split("|") if x else [], default="" 

34) 

35 

36# This shared_processors approach is modified from structlog's 

37# documentation for how to handle non-structured logs in a structured format 

38# https://www.structlog.org/en/stable/standard-library.html#rendering-using-structlog-based-formatters-within-logging 

39timestamper = structlog.processors.TimeStamper(fmt="iso") 

40 

41shared_processors = [ 

42 structlog.contextvars.merge_contextvars, 

43 structlog.stdlib.add_logger_name, 

44 structlog.stdlib.add_log_level, 

45 structlog.stdlib.PositionalArgumentsFormatter(), 

46 structlog.processors.CallsiteParameterAdder( 

47 { 

48 structlog.processors.CallsiteParameter.FILENAME, 

49 structlog.processors.CallsiteParameter.FUNC_NAME, 

50 structlog.processors.CallsiteParameter.LINENO, 

51 } 

52 ), 

53 structlog.processors.StackInfoRenderer(), 

54 # https://github.com/kiwicom/structlog-sentry - for Sentry integration 

55 # Must go after `add_logger_name` and `add_log_level`, 

56 # but before `format_exc_info` 

57 SentryProcessor( 

58 event_level=logging.ERROR, 

59 # `level` here dictates which log levels will be included in breadcrumbs 

60 # Set to WARNING for now to prevent PII/sensitive info from appearing there 

61 level=logging.WARNING, 

62 # Also ignore any Django SQL logs entirely in case they get enabled 

63 ignore_loggers=["django.db.backends"], 

64 ), 

65 structlog.processors.format_exc_info, 

66 structlog.processors.UnicodeDecoder(), 

67] 

68 

69 

70# These loggers duplicate django-structlog's request start/finish 

71# logs, as well as our nginx request logs. We want to keep only the 

72# start/finish logs and the nginx logs for access logging, otherwise 

73# we are just duplicating information! 

74_UNWANTED_LOGGERS = { 

75 "uvicorn.access", 

76 "django.request", 

77} 

78 

79 

80def suppress_unwanted_logs(record: LogRecord) -> bool: 

81 return ( 

82 record.name not in _UNWANTED_LOGGERS 

83 and "GET /healthcheck" not in record.getMessage() 

84 ) 

85 

86 

87# Logging configuration 

88LOGGING = { 

89 "version": 1, 

90 "disable_existing_loggers": False, 

91 "filters": { 

92 "suppress_unwanted_logs": { 

93 "()": "django.utils.log.CallbackFilter", 

94 "callback": suppress_unwanted_logs, 

95 }, 

96 }, 

97 "formatters": { 

98 "structured": { 

99 "()": structlog.stdlib.ProcessorFormatter, 

100 "processor": ( 

101 structlog.processors.JSONRenderer() 

102 if LOG_PROCESSOR == "json" 

103 else structlog.dev.ConsoleRenderer() 

104 ), 

105 "foreign_pre_chain": [ 

106 timestamper, 

107 # Explanations from https://www.structlog.org/en/stable/standard-library.html#rendering-using-structlog-based-formatters-within-logging 

108 # Add the log level and a timestamp to the event_dict if the log entry 

109 # is not from structlog. 

110 structlog.stdlib.add_log_level, 

111 # Add extra attributes of LogRecord objects to the event dictionary 

112 # so that values passed in the extra parameter of log methods pass 

113 # through to log output. 

114 structlog.stdlib.ExtraAdder(), 

115 ] 

116 + shared_processors, 

117 }, 

118 }, 

119 "handlers": { 

120 "console_structured": { 

121 "level": DJANGO_LOG_LEVEL, 

122 "class": "logging.StreamHandler", 

123 "formatter": "structured", 

124 "filters": ["suppress_unwanted_logs"], 

125 }, 

126 }, 

127 "root": { 

128 "handlers": ["console_structured"], 

129 "level": DJANGO_LOG_LEVEL, 

130 "propagate": False, 

131 }, 

132 "loggers": { 

133 "django": { 

134 "handlers": ["console_structured"], 

135 # Keep this at info to avoid django internal debug logs; 

136 # we just want our own debug logs when log level is set to debug 

137 "level": "INFO", 

138 "propagate": False, 

139 }, 

140 "uvicorn": { 

141 "handlers": ["console_structured"], 

142 "level": DJANGO_LOG_LEVEL, 

143 }, 

144 }, 

145} 

146 

147# https://django-structlog.readthedocs.io/en/latest/getting_started.html 

148structlog.configure( 

149 processors=[ 

150 timestamper, 

151 structlog.stdlib.filter_by_level, 

152 ] 

153 + shared_processors 

154 + [ 

155 structlog.stdlib.ProcessorFormatter.wrap_for_formatter, 

156 ], 

157 logger_factory=structlog.stdlib.LoggerFactory(), 

158 cache_logger_on_first_use=(ENVIRONMENT == "production"), 

159) 

160 

161if DJANGO_DATABASE_LOGGING: 161 ↛ 164line 161 didn't jump to line 164 because the condition on line 161 was never true

162 # Behind a separate flag as it's a very noisy debug logger, and it's nice to be 

163 # able to enable it conditionally within that context 

164 LOGGING["loggers"]["django.db.backends"] = { 

165 "level": "DEBUG", 

166 "handlers": ["console_structured"], 

167 "propagate": False, 

168 } 

169 

170 # Add connection pool logging 

171 LOGGING["loggers"]["django.db.backends.pool"] = { 

172 "level": "DEBUG", 

173 "handlers": ["console_structured"], 

174 "propagate": False, 

175 } 

176 

177 if not DEBUG: 

178 # WARNING: Do not run in production long-term as it can impact performance. 

179 middleware = ( 

180 "api.middleware.force_debug_cursor_middleware.force_debug_cursor_middleware" 

181 ) 

182 if middleware not in MIDDLEWARE: 

183 MIDDLEWARE.append(middleware)