Support serializable sql panel

Author: tim-schilling

debug_toolbar/panels/sql/forms.py

@@ -4,25 +4,22 @@
 from django.core.exceptions import ValidationError
 from django.db import connections
 from django.utils.functional import cached_property
+from django.utils.translation import gettext_lazy as _
 
 from debug_toolbar.panels.sql.utils import reformat_sql
+from debug_toolbar.toolbar import DebugToolbar
 
 
 class SQLSelectForm(forms.Form):
     """
     Validate params
 
-        sql: The sql statement with interpolated params
-        raw_sql: The sql statement with placeholders
-        params: JSON encoded parameter values
-        duration: time for SQL to execute passed in from toolbar just for redisplay
+        request_id: The identifier for the request
+        query_id: The identifier for the query
     """
 
-    sql = forms.CharField()
-    raw_sql = forms.CharField()
-    params = forms.CharField()
-    alias = forms.CharField(required=False, initial="default")
-    duration = forms.FloatField()
+    request_id = forms.CharField()
+    djdt_query_id = forms.CharField()
 
     def clean_raw_sql(self):
         value = self.cleaned_data["raw_sql"]
@@ -48,12 +45,89 @@ def clean_alias(self):
 
         return value
 
+    def clean(self):
+        cleaned_data = super().clean()
+        toolbar = DebugToolbar.fetch(
+            self.cleaned_data["request_id"], panel_id="SQLPanel"
+        )
+        if toolbar is None:
+            raise ValidationError(_("Data for this panel isn't available anymore."))
+
+        panel = toolbar.get_panel_by_id("SQLPanel")
+        # Find the query for this form submission
+        query = None
+        for q in panel.get_stats()["queries"]:
+            if q["djdt_query_id"] != self.cleaned_data["djdt_query_id"]:
+                continue
+            else:
+                query = q
+                break
+        if not query:
+            raise ValidationError(_("Invalid query id."))
+        cleaned_data["query"] = query
+        return cleaned_data
+
+    def select(self):
+        query = self.cleaned_data["query"]
+        sql = query["raw_sql"]
+        params = json.loads(query["params"])
+        with self.cursor as cursor:
+            cursor.execute(sql, params)
+            headers = [d[0] for d in cursor.description]
+            result = cursor.fetchall()
+            return result, headers
+
+    def explain(self):
+        query = self.cleaned_data["query"]
+        sql = query["raw_sql"]
+        params = json.loads(query["params"])
+        vendor = query["vendor"]
+        with self.cursor as cursor:
+            if vendor == "sqlite":
+                # SQLite's EXPLAIN dumps the low-level opcodes generated for a query;
+                # EXPLAIN QUERY PLAN dumps a more human-readable summary
+                # See https://www.sqlite.org/lang_explain.html for details
+                cursor.execute(f"EXPLAIN QUERY PLAN {sql}", params)
+            elif vendor == "postgresql":
+                cursor.execute(f"EXPLAIN ANALYZE {sql}", params)
+            else:
+                cursor.execute(f"EXPLAIN {sql}", params)
+            headers = [d[0] for d in cursor.description]
+            result = cursor.fetchall()
+            return result, headers
+
+    def profile(self):
+        query = self.cleaned_data["query"]
+        sql = query["raw_sql"]
+        params = json.loads(query["params"])
+        with self.cursor as cursor:
+            cursor.execute("SET PROFILING=1")  # Enable profiling
+            cursor.execute(sql, params)  # Execute SELECT
+            cursor.execute("SET PROFILING=0")  # Disable profiling
+            # The Query ID should always be 1 here but I'll subselect to get
+            # the last one just in case...
+            cursor.execute(
+                """
+                SELECT *
+                FROM information_schema.profiling
+                WHERE query_id = (
+                    SELECT query_id
+                    FROM information_schema.profiling
+                    ORDER BY query_id DESC
+                    LIMIT 1
+                )
+                """
+            )
+            headers = [d[0] for d in cursor.description]
+            result = cursor.fetchall()
+            return result, headers
+
     def reformat_sql(self):
-        return reformat_sql(self.cleaned_data["sql"], with_toggle=False)
+        return reformat_sql(self.cleaned_data["query"]["sql"], with_toggle=False)
 
     @property
     def connection(self):
-        return connections[self.cleaned_data["alias"]]
+        return connections[self.cleaned_data["query"]["alias"]]
 
     @cached_property
     def cursor(self):

debug_toolbar/panels/sql/panel.py

@@ -1,9 +1,10 @@
 import uuid
 from collections import defaultdict
-from copy import copy
 
 from django.db import connections
+from django.template.loader import render_to_string
 from django.urls import path
+from django.utils.functional import cached_property
 from django.utils.translation import gettext_lazy as _, ngettext
 
 from debug_toolbar import settings as dt_settings
@@ -81,7 +82,7 @@ def _similar_query_key(query):
 
 
 def _duplicate_query_key(query):
-    raw_params = () if query["raw_params"] is None else tuple(query["raw_params"])
+    raw_params = () if query["params"] is None else tuple(query["params"])
     # repr() avoids problems because of unhashable types
     # (e.g. lists) when used as dictionary keys.
     # https://github.com/jazzband/django-debug-toolbar/issues/1091
@@ -139,6 +140,7 @@ def current_transaction_id(self, alias):
         return trans_id
 
     def record(self, **kwargs):
+        kwargs["djdt_query_id"] = uuid.uuid4().hex
         self._queries.append(kwargs)
         alias = kwargs["alias"]
         if alias not in self._databases:
@@ -197,8 +199,6 @@ def disable_instrumentation(self):
             connection._djdt_logger = None
 
     def generate_stats(self, request, response):
-        colors = contrasting_color_generator()
-        trace_colors = defaultdict(lambda: next(colors))
         similar_query_groups = defaultdict(list)
         duplicate_query_groups = defaultdict(list)
 
@@ -255,14 +255,6 @@ def generate_stats(self, request, response):
                     query["trans_status"] = get_transaction_status_display(
                         query["vendor"], query["trans_status"]
                     )
-
-                query["form"] = SignedDataForm(
-                    auto_id=None, initial=SQLSelectForm(initial=copy(query)).initial
-                )
-
-                if query["sql"]:
-                    query["sql"] = reformat_sql(query["sql"], with_toggle=True)
-
                 query["is_slow"] = query["duration"] > sql_warning_threshold
                 query["is_select"] = (
                     query["raw_sql"].lower().lstrip().startswith("select")
@@ -276,9 +268,6 @@ def generate_stats(self, request, response):
                 query["start_offset"] = width_ratio_tally
                 query["end_offset"] = query["width_ratio"] + query["start_offset"]
                 width_ratio_tally += query["width_ratio"]
-                query["stacktrace"] = render_stacktrace(query["stacktrace"])
-
-                query["trace_color"] = trace_colors[query["stacktrace"]]
 
                 last_by_alias[alias] = query
 
@@ -311,3 +300,38 @@ def generate_server_timing(self, request, response):
         title = "SQL {} queries".format(len(stats.get("queries", [])))
         value = stats.get("sql_time", 0)
         self.record_server_timing("sql_time", title, value)
+
+    def record_stats(self, stats):
+        """
+        Store data gathered by the panel. ``stats`` is a :class:`dict`.
+
+        Each call to ``record_stats`` updates the statistics dictionary.
+        """
+        for query in stats.get("queries", []):
+            query["params"]
+        return super().record_stats(stats)
+
+    # Cache the content property since it manipulates the queries in the stats
+    # This allows the caller to treat content as idempotent
+    @cached_property
+    def content(self):
+        if self.has_content:
+            stats = self.get_stats()
+            colors = contrasting_color_generator()
+            trace_colors = defaultdict(lambda: next(colors))
+
+            for query in stats.get("queries", []):
+                query["sql"] = reformat_sql(query["sql"], with_toggle=True)
+                query["form"] = SignedDataForm(
+                    auto_id=None,
+                    initial=SQLSelectForm(
+                        initial={
+                            "djdt_query_id": query["djdt_query_id"],
+                            "request_id": self.toolbar.request_id,
+                        }
+                    ).initial,
+                )
+                query["stacktrace"] = render_stacktrace(query["stacktrace"])
+                query["trace_color"] = trace_colors[query["stacktrace"]]
+
+            return render_to_string(self.template, stats)

debug_toolbar/panels/sql/tracking.py

@@ -201,7 +201,6 @@ def _record(self, method, sql, params):
                 "duration": duration,
                 "raw_sql": sql,
                 "params": _params,
-                "raw_params": params,
                 "stacktrace": get_stack_trace(skip=2),
                 "template_info": template_info,
             }

debug_toolbar/panels/sql/views.py

@@ -5,6 +5,7 @@
 from debug_toolbar.decorators import render_with_toolbar_language, require_show_toolbar
 from debug_toolbar.forms import SignedDataForm
 from debug_toolbar.panels.sql.forms import SQLSelectForm
+from debug_toolbar.panels.sql.utils import reformat_sql
 
 
 def get_signed_data(request):
@@ -27,19 +28,14 @@ def sql_select(request):
     form = SQLSelectForm(verified_data)
 
     if form.is_valid():
-        sql = form.cleaned_data["raw_sql"]
-        params = form.cleaned_data["params"]
-        with form.cursor as cursor:
-            cursor.execute(sql, params)
-            headers = [d[0] for d in cursor.description]
-            result = cursor.fetchall()
-
+        query = form.cleaned_data["query"]
+        result, headers = form.select()
         context = {
             "result": result,
-            "sql": form.reformat_sql(),
-            "duration": form.cleaned_data["duration"],
+            "sql": reformat_sql(query["sql"], with_toggle=False),
+            "duration": query["duration"],
             "headers": headers,
-            "alias": form.cleaned_data["alias"],
+            "alias": query["alias"],
         }
         content = render_to_string("debug_toolbar/panels/sql_select.html", context)
         return JsonResponse({"content": content})
@@ -57,28 +53,14 @@ def sql_explain(request):
     form = SQLSelectForm(verified_data)
 
     if form.is_valid():
-        sql = form.cleaned_data["raw_sql"]
-        params = form.cleaned_data["params"]
-        vendor = form.connection.vendor
-        with form.cursor as cursor:
-            if vendor == "sqlite":
-                # SQLite's EXPLAIN dumps the low-level opcodes generated for a query;
-                # EXPLAIN QUERY PLAN dumps a more human-readable summary
-                # See https://www.sqlite.org/lang_explain.html for details
-                cursor.execute(f"EXPLAIN QUERY PLAN {sql}", params)
-            elif vendor == "postgresql":
-                cursor.execute(f"EXPLAIN ANALYZE {sql}", params)
-            else:
-                cursor.execute(f"EXPLAIN {sql}", params)
-            headers = [d[0] for d in cursor.description]
-            result = cursor.fetchall()
-
+        query = form.cleaned_data["query"]
+        result, headers = form.explain()
         context = {
             "result": result,
-            "sql": form.reformat_sql(),
-            "duration": form.cleaned_data["duration"],
+            "sql": reformat_sql(query["sql"], with_toggle=False),
+            "duration": query["duration"],
             "headers": headers,
-            "alias": form.cleaned_data["alias"],
+            "alias": query["alias"],
         }
         content = render_to_string("debug_toolbar/panels/sql_explain.html", context)
         return JsonResponse({"content": content})
@@ -96,45 +78,25 @@ def sql_profile(request):
     form = SQLSelectForm(verified_data)
 
     if form.is_valid():
-        sql = form.cleaned_data["raw_sql"]
-        params = form.cleaned_data["params"]
+        query = form.cleaned_data["query"]
         result = None
         headers = None
         result_error = None
-        with form.cursor as cursor:
-            try:
-                cursor.execute("SET PROFILING=1")  # Enable profiling
-                cursor.execute(sql, params)  # Execute SELECT
-                cursor.execute("SET PROFILING=0")  # Disable profiling
-                # The Query ID should always be 1 here but I'll subselect to get
-                # the last one just in case...
-                cursor.execute(
-                    """
-                    SELECT *
-                    FROM information_schema.profiling
-                    WHERE query_id = (
-                        SELECT query_id
-                        FROM information_schema.profiling
-                        ORDER BY query_id DESC
-                        LIMIT 1
-                    )
-                    """
-                )
-                headers = [d[0] for d in cursor.description]
-                result = cursor.fetchall()
-            except Exception:
-                result_error = (
-                    "Profiling is either not available or not supported by your "
-                    "database."
-                )
+        try:
+            result, headers = form.profile()
+        except Exception:
+            result_error = (
+                "Profiling is either not available or not supported by your "
+                "database."
+            )
 
         context = {
             "result": result,
             "result_error": result_error,
             "sql": form.reformat_sql(),
-            "duration": form.cleaned_data["duration"],
+            "duration": query["duration"],
             "headers": headers,
-            "alias": form.cleaned_data["alias"],
+            "alias": query["alias"],
         }
         content = render_to_string("debug_toolbar/panels/sql_profile.html", context)
         return JsonResponse({"content": content})

docs/changes.rst

@@ -17,6 +17,12 @@ Serializable (don't include in main)
 * Created a ``StoredDebugToolbar`` that support creating an instance of the
   toolbar representing an old request. It should only be used for fetching
   panels' contents.
+* Drop ``raw_params`` from query data.
+* Queries now have a unique ``djdt_query_id``. The SQL forms now reference
+  this id and avoid passing SQL to be executed.
+* Move the formatting logic of SQL queries to just before rendering in
+  ``SQLPanel.content``.
+
 
 Pending
 -------