Async compatible StaticFilesPanel (#1983)

* incoperate signals and contextvars to record staticfiles for concurrent requests

* remove used_static_files contextvar and its dependencies allow on app ready monkey patching

* async static files panel test

* update doc

* Code review changes

* suggested changes

Author: salty-ivy

debug_toolbar/panels/staticfiles.py

@@ -1,9 +1,11 @@
 import contextlib
+import uuid
 from contextvars import ContextVar
 from os.path import join, normpath
 
 from django.conf import settings
 from django.contrib.staticfiles import finders, storage
+from django.dispatch import Signal
 from django.utils.functional import LazyObject
 from django.utils.translation import gettext_lazy as _, ngettext
 
@@ -28,8 +30,10 @@ def url(self):
         return storage.staticfiles_storage.url(self.path)
 
 
-# This will collect the StaticFile instances across threads.
-used_static_files = ContextVar("djdt_static_used_static_files")
+# This will record and map the StaticFile instances with its associated
+# request across threads and async concurrent requests state.
+request_id_context_var = ContextVar("djdt_request_id_store")
+record_static_file_signal = Signal()
 
 
 class DebugConfiguredStorage(LazyObject):
@@ -59,7 +63,12 @@ def url(self, path):
                     # The ContextVar wasn't set yet. Since the toolbar wasn't properly
                     # configured to handle this request, we don't need to capture
                     # the static file.
-                    used_static_files.get().append(StaticFile(path))
+                    request_id = request_id_context_var.get()
+                    record_static_file_signal.send(
+                        sender=self,
+                        staticfile=StaticFile(path),
+                        request_id=request_id,
+                    )
                 return super().url(path)
 
         self._wrapped = DebugStaticFilesStorage()
@@ -73,6 +82,7 @@ class StaticFilesPanel(panels.Panel):
     A panel to display the found staticfiles.
     """
 
+    is_async = True
     name = "Static files"
     template = "debug_toolbar/panels/staticfiles.html"
 
@@ -87,12 +97,28 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.num_found = 0
         self.used_paths = []
+        self.request_id = str(uuid.uuid4())
 
-    def enable_instrumentation(self):
+    @classmethod
+    def ready(cls):
         storage.staticfiles_storage = DebugConfiguredStorage()
 
+    def _store_static_files_signal_handler(self, sender, staticfile, **kwargs):
+        # Only record the static file if the request_id matches the one
+        # that was used to create the panel.
+        # as sender of the signal and this handler will have multiple
+        # concurrent connections and we want to avoid storing of same
+        # staticfile from other connections as well.
+        if request_id_context_var.get() == self.request_id:
+            self.used_paths.append(staticfile)
+
+    def enable_instrumentation(self):
+        self.ctx_token = request_id_context_var.set(self.request_id)
+        record_static_file_signal.connect(self._store_static_files_signal_handler)
+
     def disable_instrumentation(self):
-        storage.staticfiles_storage = _original_storage
+        record_static_file_signal.disconnect(self._store_static_files_signal_handler)
+        request_id_context_var.reset(self.ctx_token)
 
     @property
     def num_used(self):
@@ -108,17 +134,6 @@ def nav_subtitle(self):
             "%(num_used)s file used", "%(num_used)s files used", num_used
         ) % {"num_used": num_used}
 
-    def process_request(self, request):
-        reset_token = used_static_files.set([])
-        response = super().process_request(request)
-        # Make a copy of the used paths so that when the
-        # ContextVar is reset, our panel still has the data.
-        self.used_paths = used_static_files.get().copy()
-        # Reset the ContextVar to be empty again, removing the reference
-        # to the list of used files.
-        used_static_files.reset(reset_token)
-        return response
-
     def generate_stats(self, request, response):
         self.record_stats(
             {

docs/architecture.rst

@@ -81,7 +81,7 @@ Problematic Parts
   the main benefit of the toolbar
 - Support for async and multi-threading: ``debug_toolbar.middleware.DebugToolbarMiddleware``
   is now async compatible and can process async requests. However certain
-  panels such as ``SQLPanel``, ``TimerPanel``,  ``StaticFilesPanel``,
+  panels such as ``SQLPanel``, ``TimerPanel``,
   ``RequestPanel``, ``HistoryPanel`` and ``ProfilingPanel`` aren't fully
   compatible and currently being worked on. For now, these panels
   are disabled by default when running in async environment.

tests/panels/test_staticfiles.py

@@ -1,5 +1,7 @@
 from django.conf import settings
 from django.contrib.staticfiles import finders
+from django.shortcuts import render
+from django.test import AsyncRequestFactory
 
 from ..base import BaseTestCase
 
@@ -27,6 +29,17 @@ def test_default_case(self):
             self.panel.get_staticfiles_dirs(), finders.FileSystemFinder().locations
         )
 
+    async def test_store_staticfiles_with_async_context(self):
+        async def get_response(request):
+            # template contains one static file
+            return render(request, "staticfiles/async_static.html")
+
+        self._get_response = get_response
+        async_request = AsyncRequestFactory().get("/")
+        response = await self.panel.process_request(async_request)
+        self.panel.generate_stats(self.request, response)
+        self.assertEqual(self.panel.num_used, 1)
+
     def test_insert_content(self):
         """
         Test that the panel only inserts content after generate_stats and

tests/templates/base.html

@@ -2,6 +2,7 @@
 <html>
     <head>
         <title>{{ title }}</title>
+        {% block head %}{% endblock %}
     </head>
     <body>
     {% block content %}{% endblock %}

tests/templates/staticfiles/async_static.html

@@ -0,0 +1,6 @@
+{% extends "base.html" %}
+{% load static %}
+
+{% block head %}
+    <link rel="stylesheet" href="{% static 'additional_static/base.css' %}">
+{% endblock %}