Added algolia search metrics to release report (boostorg#1889) (boostorg#1914)

Author: daveoconnor

config/settings.py

@@ -75,6 +75,7 @@
     "health_check.db",
     "health_check.contrib.celery",
     "imagekit",
+    "django_countries",
     # Allows authentication for Mailman
     "oauth2_provider",
     # Allauth dependencies:
@@ -584,3 +585,9 @@
 BOOST_BRANCHES = ["master", "develop"]
 OPENROUTER_URL = "https://openrouter.ai/api/v1"
 OPENROUTER_API_KEY = env("OPENROUTER_API_KEY")
+
+ALGOLIA = {
+    "app_id": env("ALGOLIA_APP_ID", None),
+    "api_key": env("ALGOLIA_API_KEY", None),
+    "region": env("ALGOLIA_APP_REGION", "us"),
+}

libraries/constants.py

@@ -365,3 +365,4 @@
 DEVELOP_RELEASE_URL_PATH_STR = "develop"
 MASTER_RELEASE_URL_PATH_STR = "master"
 VERSION_SLUG_PREFIX = "boost-"
+RELEASE_REPORT_SEARCH_TOP_COUNTRIES_LIMIT = 5

libraries/forms.py

@@ -1,3 +1,4 @@
+from structlog import get_logger
 from functools import cached_property
 from itertools import groupby, chain
 from operator import attrgetter
@@ -9,13 +10,20 @@
 from django.db.models import F, Q, Count, OuterRef, Sum, When, Value, Case
 from django.forms import Form, ModelChoiceField, ModelForm, BooleanField
 
+from algoliasearch.analytics.client import AnalyticsClientSync
+
+from config import settings
 from core.models import RenderedContent
 from reports.generation import (
     generate_wordcloud,
     get_mailing_list_post_stats,
     get_new_subscribers_stats,
+    generate_mailinglist_words,
+    generate_algolia_words,
+    get_algolia_search_stats,
 )
 from slack.models import Channel, SlackActivityBucket, SlackUser
+from versions.exceptions import BoostImportedDataException
 from versions.models import Version, ReportConfiguration
 from .models import (
     Commit,
@@ -29,6 +37,8 @@
 from mailing_list.models import EmailData
 from .utils import batched, conditional_batched
 
+logger = get_logger(__name__)
+
 
 class LibraryForm(ModelForm):
     class Meta:
@@ -661,9 +671,13 @@ def _get_slack_stats_for_channels(
         }
 
     def _get_dependency_data(self, library_order, version):
-        diffs_by_id = {
-            x["library_id"]: x for x in version.get_dependency_diffs().values()
-        }
+        try:
+            dependency_diff_values = version.get_dependency_diffs().values()
+        except BoostImportedDataException as e:
+            logger.warning(f"Could not get dependency diffs for version {version}: {e}")
+            dependency_diff_values = {}
+
+        diffs_by_id = {x["library_id"]: x for x in dependency_diff_values}
         diffs = []
         for lib_id in library_order:
             diffs.append(diffs_by_id.get(lib_id, {}))
@@ -697,9 +711,13 @@ def get_library_data(self, libraries, library_order, prior_version, version):
     def get_stats(self):
         report_configuration = self.cleaned_data["report_configuration"]
         version = Version.objects.filter(name=report_configuration.version).first()
+        # NOTE TO FUTURE DEVS: remember to account for the fact that a report
+        #  configuration may not match with a real version in frequent cases where
+        #  reports are generated before the release version has been created.
+        report_before_release = False if version else True
 
         prior_version = None
-        if not version:
+        if report_before_release:
             # if the version is not set then the user has chosen a report configuration
             #  that's not matching a live version, so we use the most recent version
             version = Version.objects.filter(name="master").first()
@@ -808,10 +826,25 @@ def get_stats(self):
                     library in [lib["library"] for lib in library_data],
                 )
             )
-        wordcloud_base64, wordcloud_top_words = generate_wordcloud(
-            version, prior_version
+        # mailinglist word cloud generation
+        mailinglist_words = generate_mailinglist_words(prior_version, version)
+        mailinglist_wordcloud_base64, mailinglist_wordcloud_top_words = (
+            generate_wordcloud(mailinglist_words, width=1400, height=700)
+        )
+
+        # algolia search word cloud generation
+        client = AnalyticsClientSync(**settings.ALGOLIA)
+        # if the report is based on a live version, look for stats for that
+        # version, otherwise use the stats for the prior (live) version
+        search_version = prior_version if report_before_release else version
+        search_list_words = generate_algolia_words(client, search_version)
+        search_wordcloud_base64, search_wordcloud_top_words = generate_wordcloud(
+            search_list_words, width=800, height=250
         )
 
+        search_stats = get_algolia_search_stats(client, search_version)
+        logger.info(f"{search_stats=}")
+
         opened_issues_count = (
             Issue.objects.filter(library__in=self.library_queryset)
             .opened_during_release(version, prior_version)
@@ -827,20 +860,23 @@ def get_stats(self):
             "committee_members": committee_members,
             "lines_added": lines_added,
             "lines_removed": lines_removed,
-            "wordcloud_base64": wordcloud_base64,
-            "wordcloud_frequencies": wordcloud_top_words,
             "version": version,
             "report_configuration": report_configuration,
             "prior_version": prior_version,
             "opened_issues_count": opened_issues_count,
             "closed_issues_count": closed_issues_count,
+            "mailinglist_wordcloud_base64": mailinglist_wordcloud_base64,
+            "mailinglist_wordcloud_frequencies": mailinglist_wordcloud_top_words,
             "mailinglist_counts": mailinglist_counts,
             "mailinglist_total": total_mailinglist_count or 0,
             "mailinglist_contributor_release_count": mailinglist_contributor_release_count,  # noqa: E501
             "mailinglist_contributor_new_count": mailinglist_contributor_new_count,
             "mailinglist_post_stats": mailinglist_post_stats,
             "mailinglist_new_subscribers_stats": new_subscribers_stats,
             "mailinglist_charts_start_year": prior_version.release_date.year,
+            "search_wordcloud_base64": search_wordcloud_base64,
+            "search_wordcloud_frequencies": search_wordcloud_top_words,
+            "search_stats": search_stats,
             "commit_contributors_release_count": commit_contributors_release_count,
             "commit_contributors_new_count": commit_contributors_new_count,
             "global_contributors_new_count": len(

reports/generation.py

@@ -1,5 +1,6 @@
 import base64
 import io
+import json
 import logging
 import random
 from datetime import datetime, timedelta, date
@@ -11,8 +12,10 @@
 from django.db.models.functions import ExtractWeek, ExtractIsoYear
 from matplotlib import pyplot as plt
 from wordcloud import WordCloud, STOPWORDS
+from algoliasearch.analytics.client import AnalyticsClientSync
 
 from core.models import SiteSettings
+from libraries.constants import RELEASE_REPORT_SEARCH_TOP_COUNTRIES_LIMIT
 from libraries.models import WordcloudMergeWord  # TODO: move model to this app
 from mailing_list.models import PostingData, SubscriptionData
 from reports.constants import WORDCLOUD_FONT
@@ -21,14 +24,45 @@
 logger = logging.getLogger(__name__)
 
 
+def generate_mailinglist_words(
+    prior_version: Version, version: Version
+) -> dict[str, int]:
+    """Generates word frequencies from mailing list content between two versions."""
+    word_frequencies = {}
+    for content in get_mail_content(version, prior_version):
+        for key, val in WordCloud().process_text(content).items():
+            if len(key) < 2:
+                continue
+            key_lower = key.lower()
+            if key_lower not in word_frequencies:
+                word_frequencies[key_lower] = 0
+            word_frequencies[key_lower] += val
+
+    return word_frequencies
+
+
+def generate_algolia_words(
+    client: AnalyticsClientSync, version: Version
+) -> dict[str, int]:
+    args = {
+        "index": version.stripped_boost_url_slug,
+        "limit": 100,
+    }
+    search_results = client.get_top_searches(**args).to_json()
+    search_data = json.loads(search_results)
+    return {r["search"]: r["count"] for r in search_data["searches"] if r["count"] > 1}
+
+
 def generate_wordcloud(
-    version: Version, prior_version: Version
+    word_frequencies: dict[str, int], width: int, height: int
 ) -> tuple[str | None, list]:
     """Generates a wordcloud png and returns it as a base64 string and word frequencies.
 
     Returns:
         Tuple of (base64_encoded_png_string, wordcloud_top_words)
     """
+    if not word_frequencies:
+        return None, []
     font_relative_path = f"font/{WORDCLOUD_FONT}"
     font_full_path = finders.find(font_relative_path)
 
@@ -38,23 +72,11 @@ def generate_wordcloud(
     wc = WordCloud(
         mode="RGBA",
         background_color=None,
-        width=1400,
-        height=700,
+        width=width,
+        height=height,
         stopwords=STOPWORDS | SiteSettings.load().wordcloud_ignore_set,
         font_path=font_full_path,
     )
-    word_frequencies = {}
-    for content in get_mail_content(version, prior_version):
-        for key, val in wc.process_text(content).items():
-            if len(key) < 2:
-                continue
-            key_lower = key.lower()
-            if key_lower not in word_frequencies:
-                word_frequencies[key_lower] = 0
-            word_frequencies[key_lower] += val
-    if not word_frequencies:
-        return None, []
-
     word_frequencies = boost_normalize_words(
         word_frequencies,
         {x.from_word: x.to_word for x in WordcloudMergeWord.objects.all()},
@@ -181,3 +203,21 @@ def get_new_subscribers_stats(start_date: datetime, end_date: datetime):
         current += timedelta(days=7)  # hop by weeks
 
     return chart_data
+
+
+def get_algolia_search_stats(client: AnalyticsClientSync, version: Version) -> dict:
+    default_args = {"index": version.stripped_boost_url_slug}
+    # search data
+    search_response = client.get_searches_count(**default_args).to_json()
+    search_data = json.loads(search_response)
+    # country data
+    country_results = client.get_top_countries(**default_args, limit=100).to_json()
+    country_data = json.loads(country_results)
+    country_stats = {r["country"]: r["count"] for r in country_data["countries"]}
+    return {
+        "total_searches": search_data.get("count"),
+        "country_stats": country_stats,
+        "top_countries": list(country_stats.items())[
+            :RELEASE_REPORT_SEARCH_TOP_COUNTRIES_LIMIT
+        ],
+    }

requirements.in

@@ -5,6 +5,7 @@ django-allauth
 django-allauth[socialaccount]
 django-anymail[mailgun]
 django-cors-headers
+django-countries
 django-db-geventpool
 django-extensions
 django-health-check
@@ -34,6 +35,7 @@ jsoncomment
 unidecode
 wordcloud
 lxml
+algoliasearch
 openai
 
 # Logging