Use database to keep track of session cleaning (#167)

- Use database to keep track of when ReactPy last performs a clean.
   - Previously done via a cache entry for performance. But realistically we should assume the user is either utilizing a performant DB, or doesn't care about performance to begin with.
- Store web module file contents in cache, instead of the HTTP Response
- Add a timeout to the web module cached files to allow old entries to be deleted.

Author: Archmonger

CHANGELOG.md

@@ -44,6 +44,7 @@ Using the following categories, list your changes in this order:
 -   If using `settings.py:REACTPY_DATABASE`, `reactpy_django.database.Router` must now be registered in `settings.py:DATABASE_ROUTERS`.
 -   By default, ReactPy will now use a backhaul thread to increase performance.
 -   Minimum Python version required is now `3.9`
+-   A thread-safe cache is no longer required.
 
 ## [3.2.1] - 2023-06-29
 

docs/python/settings.py

@@ -1,5 +1,5 @@
 # Cache used to store ReactPy web modules.
-# ReactPy requires a multiprocessing-safe and thread-safe cache.
+# ReactPy benefits from a fast, well indexed cache.
 # We recommend redis or python-diskcache.
 REACTPY_CACHE = "default"
 

docs/src/contribute/running-tests.md

@@ -41,5 +41,5 @@ Alternatively, if you want to only run Django related tests, you can use the fol
 
 ```bash linenums="0"
 cd tests
-python mange.py test
+python manage.py test
 ```

src/reactpy_django/checks.py

@@ -1,3 +1,6 @@
+import sys
+
+from django.contrib.staticfiles.finders import find
 from django.core.checks import Error, Tags, Warning, register
 
 
@@ -6,6 +9,8 @@ def reactpy_warnings(app_configs, **kwargs):
     from django.conf import settings
     from django.urls import reverse
 
+    from reactpy_django.config import REACTPY_FAILED_COMPONENTS
+
     warnings = []
 
     # REACTPY_DATABASE is not an in-memory database.
@@ -25,23 +30,6 @@ def reactpy_warnings(app_configs, **kwargs):
             )
         )
 
-    # REACTPY_CACHE is not an in-memory cache.
-    if getattr(settings, "CACHES", {}).get(
-        getattr(settings, "REACTPY_CACHE", "default"), {}
-    ).get("BACKEND", None) in {
-        "django.core.cache.backends.dummy.DummyCache",
-        "django.core.cache.backends.locmem.LocMemCache",
-    }:
-        warnings.append(
-            Warning(
-                "Using ReactPy with an in-memory cache can cause unexpected "
-                "behaviors.",
-                hint="Configure settings.py:CACHES[REACTPY_CACHE], to use a "
-                "multiprocessing and thread safe cache.",
-                id="reactpy_django.W002",
-            )
-        )
-
     # ReactPy URLs exist
     try:
         reverse("reactpy:web_modules", kwargs={"file": "example"})
@@ -52,10 +40,47 @@ def reactpy_warnings(app_configs, **kwargs):
                 "ReactPy URLs have not been registered.",
                 hint="""Add 'path("reactpy/", include("reactpy_django.http.urls"))' """
                 "to your application's urlpatterns.",
+                id="reactpy_django.W002",
+            )
+        )
+
+    # Warn if REACTPY_BACKHAUL_THREAD is set to True on Linux with Daphne
+    if (
+        sys.argv
+        and sys.argv[0].endswith("daphne")
+        and getattr(settings, "REACTPY_BACKHAUL_THREAD", True)
+        and sys.platform == "linux"
+    ):
+        warnings.append(
+            Warning(
+                "REACTPY_BACKHAUL_THREAD is enabled but you running with Daphne on Linux. "
+                "This configuration is known to be unstable.",
+                hint="Set settings.py:REACTPY_BACKHAUL_THREAD to False or use a different webserver.",
                 id="reactpy_django.W003",
             )
         )
 
+    # Check if reactpy_django/client.js is available
+    if not find("reactpy_django/client.js"):
+        warnings.append(
+            Warning(
+                "ReactPy client.js could not be found within Django static files!",
+                hint="Check your Django static file configuration.",
+                id="reactpy_django.W004",
+            )
+        )
+
+    # Check if any components failed to be registered
+    if REACTPY_FAILED_COMPONENTS:
+        warnings.append(
+            Warning(
+                "ReactPy failed to register the following components:\n\t+ "
+                + "\n\t+ ".join(REACTPY_FAILED_COMPONENTS),
+                hint="Check if these paths are valid, or if an exception is being raised during import.",
+                id="reactpy_django.W005",
+            )
+        )
+
     return warnings
 
 
@@ -69,8 +94,7 @@ def reactpy_errors(app_configs, **kwargs):
     if not getattr(settings, "ASGI_APPLICATION", None):
         errors.append(
             Error(
-                "ASGI_APPLICATION is not defined."
-                " ReactPy requires ASGI to be enabled.",
+                "ASGI_APPLICATION is not defined, but ReactPy requires ASGI.",
                 hint="Add ASGI_APPLICATION to settings.py.",
                 id="reactpy_django.E001",
             )
@@ -150,4 +174,14 @@ def reactpy_errors(app_configs, **kwargs):
             )
         )
 
+    # Check for dependencies
+    if "channels" not in settings.INSTALLED_APPS:
+        errors.append(
+            Error(
+                "Django Channels is not installed.",
+                hint="Add 'channels' to settings.py:INSTALLED_APPS.",
+                id="reactpy_django.E009",
+            )
+        )
+
     return errors

src/reactpy_django/components.py

@@ -210,7 +210,6 @@ def _cached_static_contents(static_path: str):
         )
 
     # Fetch the file from cache, if available
-    # Cache is preferrable to `use_memo` due to multiprocessing capabilities
     last_modified_time = os.stat(abs_path).st_mtime
     cache_key = f"reactpy_django:static_contents:{static_path}"
     file_contents = caches[REACTPY_CACHE].get(

src/reactpy_django/config.py

@@ -1,8 +1,5 @@
 from __future__ import annotations
 
-import logging
-import sys
-
 from django.conf import settings
 from django.core.cache import DEFAULT_CACHE_ALIAS
 from django.db import DEFAULT_DB_ALIAS
@@ -16,12 +13,10 @@
 )
 from reactpy_django.utils import import_dotted_path
 
-_logger = logging.getLogger(__name__)
-
-
 # Non-configurable values
 REACTPY_DEBUG_MODE.set_current(getattr(settings, "DEBUG"))
 REACTPY_REGISTERED_COMPONENTS: dict[str, ComponentConstructor] = {}
+REACTPY_FAILED_COMPONENTS: set[str] = set()
 REACTPY_VIEW_COMPONENT_IFRAMES: dict[str, ViewComponentIframe] = {}
 
 
@@ -68,16 +63,3 @@
     "REACTPY_BACKHAUL_THREAD",
     True,
 )
-
-# Settings checks (separate from Django checks)
-if (
-    sys.platform == "linux"
-    and sys.argv
-    and sys.argv[0].endswith("daphne")
-    and REACTPY_BACKHAUL_THREAD
-):
-    _logger.warning(
-        "ReactPy is running on Linux with Daphne, but REACTPY_BACKHAUL_THREAD is set "
-        "to True. This configuration is known to be unstable. Either set "
-        "REACTPY_BACKHAUL_THREAD to False, or run ReactPy with a different ASGI server."
-    )

src/reactpy_django/http/views.py

@@ -15,7 +15,7 @@ async def web_modules_file(request: HttpRequest, file: str) -> HttpResponse:
     from reactpy_django.config import REACTPY_CACHE
 
     web_modules_dir = REACTPY_WEB_MODULES_DIR.current
-    path = os.path.abspath(web_modules_dir.joinpath(*file.split("/")))
+    path = os.path.abspath(web_modules_dir.joinpath(file))
 
     # Prevent attempts to walk outside of the web modules dir
     if str(web_modules_dir) != os.path.commonpath((path, web_modules_dir)):
@@ -25,18 +25,18 @@ async def web_modules_file(request: HttpRequest, file: str) -> HttpResponse:
 
     # Fetch the file from cache, if available
     last_modified_time = os.stat(path).st_mtime
-    cache_key = create_cache_key("web_module", str(path).lstrip(str(web_modules_dir)))
-    response = await caches[REACTPY_CACHE].aget(
+    cache_key = create_cache_key("web_modules", path)
+    file_contents = await caches[REACTPY_CACHE].aget(
         cache_key, version=int(last_modified_time)
     )
-    if response is None:
+    if file_contents is None:
         async with async_open(path, "r") as fp:
-            response = HttpResponse(await fp.read(), content_type="text/javascript")
+            file_contents = await fp.read()
         await caches[REACTPY_CACHE].adelete(cache_key)
         await caches[REACTPY_CACHE].aset(
-            cache_key, response, timeout=None, version=int(last_modified_time)
+            cache_key, file_contents, timeout=604800, version=int(last_modified_time)
         )
-    return response
+    return HttpResponse(file_contents, content_type="text/javascript")
 
 
 async def view_to_component_iframe(

src/reactpy_django/migrations/0004_config.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.2.3 on 2023-08-04 05:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("reactpy_django", "0003_componentsession_delete_componentparams"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Config",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("cleaned_at", models.DateTimeField(auto_now_add=True)),
+            ],
+        ),
+    ]

src/reactpy_django/models.py

@@ -9,3 +9,19 @@ class ComponentSession(models.Model):
     uuid = models.UUIDField(primary_key=True, editable=False, unique=True)  # type: ignore
     params = models.BinaryField(editable=False)  # type: ignore
     last_accessed = models.DateTimeField(auto_now_add=True)  # type: ignore
+
+
+class Config(models.Model):
+    """A singleton model for storing ReactPy configuration."""
+
+    cleaned_at = models.DateTimeField(auto_now_add=True)  # type: ignore
+
+    def save(self, *args, **kwargs):
+        """Singleton save method."""
+        self.pk = 1
+        super().save(*args, **kwargs)
+
+    @classmethod
+    def load(cls):
+        obj, created = cls.objects.get_or_create(pk=1)
+        return obj

src/reactpy_django/utils.py

@@ -5,14 +5,13 @@
 import logging
 import os
 import re
-from datetime import datetime, timedelta
+from datetime import timedelta
 from fnmatch import fnmatch
 from importlib import import_module
 from inspect import iscoroutinefunction
 from typing import Any, Callable, Sequence
 
 from channels.db import database_sync_to_async
-from django.core.cache import caches
 from django.db.models import ManyToManyField, ManyToOneRel, prefetch_related_objects
 from django.db.models.base import Model
 from django.db.models.query import QuerySet
@@ -24,7 +23,6 @@
 
 from reactpy_django.exceptions import ComponentDoesNotExistError, ComponentParamError
 
-
 _logger = logging.getLogger(__name__)
 _component_tag = r"(?P<tag>component)"
 _component_path = r"(?P<path>\"[^\"'\s]+\"|'[^\"'\s]+')"
@@ -88,14 +86,18 @@ def _register_component(dotted_path: str) -> Callable:
     """Adds a component to the mapping of registered components.
     This should only be called on startup to maintain synchronization during mulitprocessing.
     """
-    from reactpy_django.config import REACTPY_REGISTERED_COMPONENTS
+    from reactpy_django.config import (
+        REACTPY_FAILED_COMPONENTS,
+        REACTPY_REGISTERED_COMPONENTS,
+    )
 
     if dotted_path in REACTPY_REGISTERED_COMPONENTS:
         return REACTPY_REGISTERED_COMPONENTS[dotted_path]
 
     try:
         REACTPY_REGISTERED_COMPONENTS[dotted_path] = import_dotted_path(dotted_path)
     except AttributeError as e:
+        REACTPY_FAILED_COMPONENTS.add(dotted_path)
         raise ComponentDoesNotExistError(
             f"Error while fetching '{dotted_path}'. {(str(e).capitalize())}."
         ) from e
@@ -266,7 +268,7 @@ def django_query_postprocessor(
             # Force the query to execute
             getattr(data, field.name, None)
 
-            if many_to_one and type(field) == ManyToOneRel:
+            if many_to_one and type(field) == ManyToOneRel:  # noqa: #E721
                 prefetch_fields.append(field.related_name or f"{field.name}_set")
 
             elif many_to_many and isinstance(field, ManyToManyField):
@@ -332,35 +334,23 @@ def create_cache_key(*args):
 def db_cleanup(immediate: bool = False):
     """Deletes expired component sessions from the database.
     This function may be expanded in the future to include additional cleanup tasks."""
-    from .config import REACTPY_CACHE, REACTPY_DEBUG_MODE, REACTPY_RECONNECT_MAX
-    from .models import ComponentSession
-
-    clean_started_at = datetime.now()
-    cache_key: str = create_cache_key("last_cleaned")
-    now_str: str = datetime.strftime(timezone.now(), DATE_FORMAT)
-    cleaned_at_str: str = caches[REACTPY_CACHE].get(cache_key)
-    cleaned_at: datetime = timezone.make_aware(
-        datetime.strptime(cleaned_at_str or now_str, DATE_FORMAT)
-    )
-    clean_needed_by = cleaned_at + timedelta(seconds=REACTPY_RECONNECT_MAX)
-    expires_by: datetime = timezone.now() - timedelta(seconds=REACTPY_RECONNECT_MAX)
+    from .config import REACTPY_DEBUG_MODE, REACTPY_RECONNECT_MAX
+    from .models import ComponentSession, Config
 
-    # Component params exist in the DB, but we don't know when they were last cleaned
-    if not cleaned_at_str and ComponentSession.objects.all():
-        _logger.warning(
-            "ReactPy has detected component sessions in the database, "
-            "but no timestamp was found in cache. This may indicate that "
-            "the cache has been cleared."
-        )
+    config = Config.load()
+    start_time = timezone.now()
+    cleaned_at = config.cleaned_at
+    clean_needed_by = cleaned_at + timedelta(seconds=REACTPY_RECONNECT_MAX)
 
     # Delete expired component parameters
-    # Use timestamps in cache (`cleaned_at_str`) as a no-dependency rate limiter
-    if immediate or not cleaned_at_str or timezone.now() >= clean_needed_by:
-        ComponentSession.objects.filter(last_accessed__lte=expires_by).delete()
-        caches[REACTPY_CACHE].set(cache_key, now_str, timeout=None)
+    if immediate or timezone.now() >= clean_needed_by:
+        expiration_date = timezone.now() - timedelta(seconds=REACTPY_RECONNECT_MAX)
+        ComponentSession.objects.filter(last_accessed__lte=expiration_date).delete()
+        config.cleaned_at = timezone.now()
+        config.save()
 
     # Check if cleaning took abnormally long
-    clean_duration = datetime.now() - clean_started_at
+    clean_duration = timezone.now() - start_time
     if REACTPY_DEBUG_MODE and clean_duration.total_seconds() > 1:
         _logger.warning(
             "ReactPy has taken %s seconds to clean up expired component sessions. "