Improved logging formatting (boostorg#1905)

Author: daveoconnor

config/celery.py

@@ -1,8 +1,11 @@
 import os
 import datetime
+from logging.config import dictConfig
 
 from celery import Celery
 from celery.schedules import crontab
+from celery.signals import setup_logging
+from pythonjsonlogger import jsonlogger
 
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
@@ -19,6 +22,40 @@
 app.autodiscover_tasks()
 
 
+@setup_logging.connect
+def config_loggers(*args, **kwags):
+    """Configure Celery logging with JSON default, readable for local development."""
+    from django.conf import settings
+
+    # production
+    json_formatter = {
+        "()": jsonlogger.JsonFormatter,
+        "format": "%(name)s %(levelname)s %(filename)s:%(lineno)d %(message)s",
+    }
+    # local development
+    text_formatter = {
+        "format": "%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d - %(message)s",
+        "datefmt": "%Y-%m-%d %H:%M:%S",
+    }
+    dictConfig(
+        {
+            "version": 1,
+            "disable_existing_loggers": False,
+            "formatters": {"json": json_formatter, "text": text_formatter},
+            "handlers": {
+                "console": {
+                    "class": "logging.StreamHandler",
+                    "formatter": "text" if settings.LOCAL_DEVELOPMENT else "json",
+                },
+            },
+            "root": {
+                "level": "INFO",
+                "handlers": ["console"],
+            },
+        }
+    )
+
+
 @app.task(bind=True)
 def debug_task(self):
     print(f"Request: {self.request!r}")

config/settings.py

@@ -251,7 +251,19 @@
 
 
 handler = logging.StreamHandler(sys.stdout)
-handler.setFormatter(jsonlogger.JsonFormatter())
+handler.setFormatter(
+    jsonlogger.JsonFormatter(
+        "%(name)s %(levelname)s %(filename)s:%(lineno)d %(message)s"
+    )
+)
+
+if LOCAL_DEVELOPMENT:
+    handler.setFormatter(
+        logging.Formatter(
+            "%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d - %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+    )
 
 root.addHandler(handler)
 

versions/management/commands/import_archives_release_data.py

@@ -3,6 +3,7 @@
 import requests
 
 from django.conf import settings
+import structlog
 
 from versions.models import Version
 from versions.releases import (
@@ -17,6 +18,8 @@
 
 logger = get_logger(__name__)
 
+logger = structlog.get_logger()
+
 
 @click.command()
 @click.option("--release", is_flag=False, help="Release name")
@@ -71,7 +74,7 @@ def command(release: str, new: bool):
                     data = get_archives_download_data(url)
                 download_data.append(data)
             except (requests.exceptions.HTTPError, ValueError):
-                print(f"Skipping {url}; error retrieving download data")
+                logger.warning(f"Skipping {url}; error retrieving download data")
                 continue
             logger.info(f"Data for {v.name=} at {url=}: {download_data=}")
             store_release_downloads_for_version(v, download_data)