Batched deletion of rendered content (boostorg#1959) (boostorg#1969)

Author: daveoconnor

core/admin.py

@@ -1,12 +1,50 @@
 from django.contrib import admin
+from django.urls import path
+from django.shortcuts import redirect, render
+from django.contrib import messages
 from .models import RenderedContent, SiteSettings
+from .tasks import delete_all_rendered_content
 
 
 @admin.register(RenderedContent)
 class RenderedContentAdmin(admin.ModelAdmin):
     list_display = ("cache_key", "content_type", "modified")
     search_fields = ("cache_key",)
 
+    def get_urls(self):
+        urls = super().get_urls()
+        custom_urls = [
+            path(
+                "delete-all/",
+                self.admin_site.admin_view(self.delete_all_view),
+                name="core_renderedcontent_delete_all",
+            ),
+        ]
+        return custom_urls + urls
+
+    def delete_all_view(self, request):
+        if request.method == "POST":
+            delete_all_rendered_content.delay()
+            messages.success(
+                request,
+                "Mass deletion task has been queued. All rendered content "
+                "records will be deleted in batches. This may take some time.",
+            )
+            return redirect("..")
+
+        context = {
+            **self.admin_site.each_context(request),
+            "title": "Delete All Rendered Content",
+        }
+        return render(
+            request, "admin/core/renderedcontent/delete_all_confirmation.html", context
+        )
+
+    def changelist_view(self, request, extra_context=None):
+        extra_context = extra_context or {}
+        extra_context["has_delete_all"] = True
+        return super().changelist_view(request, extra_context=extra_context)
+
 
 @admin.register(SiteSettings)
 class SiteSettingsAdmin(admin.ModelAdmin):

core/constants.py

@@ -69,3 +69,4 @@ class SourceDocType(Enum):
     "master/libs/redis",
     "doc/antora/url",
 ]
+RENDERED_CONTENT_BATCH_DELETE_SIZE = 10000

core/tasks.py

@@ -7,6 +7,7 @@
 
 from core.asciidoc import convert_adoc_to_html
 from .boostrenderer import get_content_from_s3
+from .constants import RENDERED_CONTENT_BATCH_DELETE_SIZE
 from .models import RenderedContent
 
 logger = structlog.get_logger()
@@ -84,3 +85,33 @@ def save_rendered_content(cache_key, content_type, content_html, last_updated_at
         obj_id=obj.id,
         obj_created=created,
     )
+
+
+@shared_task
+def delete_all_rendered_content():
+    """
+    Deletes all RenderedContent objects, in batches to avoid locking the entire table.
+    """
+    from django.db import connection
+
+    deleted_count = 0
+
+    while True:
+        pks = RenderedContent.objects.values_list("pk", flat=True)[
+            :RENDERED_CONTENT_BATCH_DELETE_SIZE
+        ]
+        if not pks:
+            break
+        batch_size, _ = RenderedContent.objects.filter(pk__in=pks).delete()
+
+        deleted_count += batch_size
+        logger.info(f"batch deleted {batch_size=} {deleted_count=}")
+
+    # Reset auto-increment sequence to 1
+    with connection.cursor() as cursor:
+        cursor.execute(
+            f"ALTER SEQUENCE {RenderedContent._meta.db_table}_id_seq RESTART WITH 1"
+        )
+
+    logger.info("all_rendered_content_deleted", total_count=deleted_count)
+    return deleted_count

templates/admin/core/renderedcontent/change_list.html

@@ -0,0 +1,13 @@
+{% extends "admin/change_list.html" %}
+{% load i18n admin_urls %}
+
+{% block object-tools-items %}
+    {{ block.super }}
+    {% if has_delete_all %}
+    <li>
+        <a href="{% url 'admin:core_renderedcontent_delete_all' %}" class="deletelink">
+            {% trans "Delete All Records" %}
+        </a>
+    </li>
+    {% endif %}
+{% endblock %}

templates/admin/core/renderedcontent/delete_all_confirmation.html

@@ -0,0 +1,29 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% block extrahead %}
+    {{ block.super }}
+    <script src="{% static 'admin/js/cancel.js' %}" async></script>
+{% endblock %}
+
+{% block bodyclass %}{{ block.super }} app-core model-renderedcontent delete-confirmation{% endblock %}
+
+{% block breadcrumbs %}
+<div class="breadcrumbs">
+<a href="{% url 'admin:index' %}">{% translate 'Home' %}</a>
+&rsaquo; <a href="{% url 'admin:core_renderedcontent_changelist' %}">{% translate 'Rendered Contents' %}</a>
+&rsaquo; {% translate 'Delete All' %}
+</div>
+{% endblock %}
+
+{% block content %}
+<p>{% translate "Are you sure you want to delete ALL rendered content records?" %}</p>
+<p><strong>{% translate "Warning:" %}</strong> {% translate "This action will queue a background task to delete all records in batches. This cannot be undone." %}</p>
+<form method="post">{% csrf_token %}
+<div>
+<input type="hidden" name="post" value="yes">
+<input type="submit" value="Yes, I'm sure">
+<a href="#" class="button cancel-link">No, take me back</a>
+</div>
+</form>
+{% endblock %}