diff --git a/config/django/base.py b/config/django/base.py index 700dcd62..dde308af 100644 --- a/config/django/base.py +++ b/config/django/base.py @@ -83,8 +83,6 @@ ROOT_URLCONF = "config.urls" -print(os.path.join(APPS_DIR, "templates")) - TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", @@ -174,6 +172,13 @@ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" +from config.settings.loggers.settings import * # noqa +from config.settings.loggers.setup import LoggersSetup # noqa + +INSTALLED_APPS, MIDDLEWARE = LoggersSetup.setup_settings(INSTALLED_APPS, MIDDLEWARE) +LoggersSetup.setup_structlog() +LOGGING = LoggersSetup.setup_logging() + from config.settings.celery import * # noqa from config.settings.cors import * # noqa from config.settings.email_sending import * # noqa @@ -189,6 +194,4 @@ INSTALLED_APPS, MIDDLEWARE = DebugToolbarSetup.do_settings(INSTALLED_APPS, MIDDLEWARE) -SHELL_PLUS_IMPORTS = [ - "from styleguide_example.blog_examples.print_qs_in_shell.utils import print_qs" -] +SHELL_PLUS_IMPORTS = ["from styleguide_example.blog_examples.print_qs_in_shell.utils import print_qs"] diff --git a/config/settings/debug_toolbar/setup.py b/config/settings/debug_toolbar/setup.py index 20bfed7c..416662b4 100644 --- a/config/settings/debug_toolbar/setup.py +++ b/config/settings/debug_toolbar/setup.py @@ -1,9 +1,5 @@ -import logging - from django.urls import include, path -logger = logging.getLogger("configuration") - def show_toolbar(*args, **kwargs) -> bool: """ @@ -30,7 +26,6 @@ def show_toolbar(*args, **kwargs) -> bool: try: import debug_toolbar # noqa except ImportError: - logger.info("No installation found for: django_debug_toolbar") return False return True @@ -44,7 +39,6 @@ class DebugToolbarSetup: @staticmethod def do_settings(INSTALLED_APPS, MIDDLEWARE, middleware_position=None): _show_toolbar: bool = show_toolbar() - logger.info(f"Django Debug Toolbar in use: {_show_toolbar}") if not _show_toolbar: return INSTALLED_APPS, MIDDLEWARE diff --git a/config/settings/loggers/settings.py b/config/settings/loggers/settings.py new file mode 100644 index 00000000..b1e0e4a7 --- /dev/null +++ b/config/settings/loggers/settings.py @@ -0,0 +1,16 @@ +import logging +from enum import Enum + +from config.env import env, env_to_enum + + +class LoggingFormat(Enum): + DEV = "dev" + JSON = "json" + LOGFMT = "logfmt" + + +LOGGING_FORMAT = env_to_enum(LoggingFormat, env("LOGGING_FORMAT", default=LoggingFormat.DEV.value)) + +DJANGO_STRUCTLOG_STATUS_4XX_LOG_LEVEL = logging.INFO +DJANGO_STRUCTLOG_CELERY_ENABLED = True diff --git a/config/settings/loggers/setup.py b/config/settings/loggers/setup.py new file mode 100644 index 00000000..9b6e87a9 --- /dev/null +++ b/config/settings/loggers/setup.py @@ -0,0 +1,141 @@ +import logging + +import structlog + + +class IgnoreFilter(logging.Filter): + def filter(self, record): + return False + + +class LoggersSetup: + """ + We use a class, just for namespacing convenience. + """ + + @staticmethod + def setup_settings(INSTALLED_APPS, MIDDLEWARE, middleware_position=None): + INSTALLED_APPS = INSTALLED_APPS + ["django_structlog"] + + django_structlog_middleware = "django_structlog.middlewares.RequestMiddleware" + + if middleware_position is None: + MIDDLEWARE = MIDDLEWARE + [django_structlog_middleware] + else: + # Grab a new copy of the list, since insert mutates the internal structure + _middleware = MIDDLEWARE[::] + _middleware.insert(middleware_position, django_structlog_middleware) + + MIDDLEWARE = _middleware + + return INSTALLED_APPS, MIDDLEWARE + + @staticmethod + def setup_structlog(): + from config.settings.loggers.settings import LOGGING_FORMAT, LoggingFormat + + logging_format = LOGGING_FORMAT + + extra_processors = [] + + if logging_format == LoggingFormat.DEV: + extra_processors = [ + structlog.processors.format_exc_info, + ] + + if logging_format in [LoggingFormat.JSON, LoggingFormat.LOGFMT]: + dict_tracebacks = structlog.processors.ExceptionRenderer( + structlog.processors.ExceptionDictTransformer(show_locals=False) + ) + extra_processors = [ + dict_tracebacks, + ] + + structlog.configure( + processors=[ + structlog.contextvars.merge_contextvars, + structlog.stdlib.filter_by_level, + structlog.processors.TimeStamper(fmt="iso", utc=True), + structlog.stdlib.add_logger_name, + structlog.stdlib.add_log_level, + structlog.stdlib.PositionalArgumentsFormatter(), + structlog.processors.StackInfoRenderer(), + structlog.dev.set_exc_info, + *extra_processors, + structlog.processors.UnicodeDecoder(), + structlog.processors.CallsiteParameterAdder( + { + structlog.processors.CallsiteParameter.FILENAME, + structlog.processors.CallsiteParameter.FUNC_NAME, + structlog.processors.CallsiteParameter.LINENO, + } + ), + structlog.stdlib.ProcessorFormatter.wrap_for_formatter, + ], + logger_factory=structlog.stdlib.LoggerFactory(), + cache_logger_on_first_use=True, + ) + + @staticmethod + def setup_logging(): + from config.settings.loggers.settings import LOGGING_FORMAT, LoggingFormat + + logging_format = LOGGING_FORMAT + formatter = "dev" + + if logging_format == LoggingFormat.DEV: + formatter = "dev" + + if logging_format == LoggingFormat.JSON: + formatter = "json" + + if logging_format == LoggingFormat.LOGFMT: + formatter = "logfmt" + + return { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "dev": { + "()": structlog.stdlib.ProcessorFormatter, + "processor": structlog.dev.ConsoleRenderer(), + }, + "logfmt": { + "()": structlog.stdlib.ProcessorFormatter, + "processor": structlog.processors.LogfmtRenderer(), + }, + "json": { + "()": structlog.stdlib.ProcessorFormatter, + "processor": structlog.processors.JSONRenderer(), + }, + }, + "filters": { + "ignore": { + "()": "config.settings.loggers.setup.IgnoreFilter", + }, + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": formatter, + } + }, + "loggers": { + # We want to get rid of the runserver logs + "django.server": {"propagate": False, "handlers": ["console"], "filters": ["ignore"]}, + # We want to get rid of the logs for 4XX and 5XX + "django.request": {"propagate": False, "handlers": ["console"], "filters": ["ignore"]}, + "django_structlog": { + "handlers": ["console"], + "level": "INFO", + }, + "celery": { + "handlers": ["console"], + "level": "INFO", + }, + "styleguide_example": { + "handlers": ["console"], + "level": "INFO", + }, + }, + } diff --git a/gunicorn.conf.py b/gunicorn.conf.py index a7596f80..7dd3c2a9 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -1,8 +1,134 @@ -# If you are not having memory issues, just delete this. -# This is primarily to prevent memory leaks -# Based on https://devcenter.heroku.com/articles/python-gunicorn -# Based on https://adamj.eu/tech/2019/09/19/working-around-memory-leaks-in-your-django-app/ -# https://docs.gunicorn.org/en/latest/settings.html#max-requests -# https://docs.gunicorn.org/en/latest/settings.html#max-requests-jitter -max_requests = 1200 -max_requests_jitter = 100 +# https://mattsegal.dev/django-gunicorn-nginx-logging.html +# https://albersdevelopment.net/2019/08/15/using-structlog-with-gunicorn/ + +import logging +import logging.config +import re + +import structlog + + +def combined_logformat(logger, name, event_dict): + if event_dict.get("logger") == "gunicorn.access": + message = event_dict["event"] + + parts = [ + r"(?P\S+)", # host %h + r"\S+", # indent %l (unused) + r"(?P\S+)", # user %u + r"\[(?P