Synced webpages from sitemap to Document model for search.

Author: sarahboyce

docs/management/commands/update_docs.py

@@ -132,6 +132,8 @@ def build_doc_release(self, release, force=False, interactive=False):
         if self.verbosity >= 1:
             self.stdout.write(f"Starting update for {release} at {datetime.now()}...")
 
+        release.sync_from_sitemap(force=force)
+
         # checkout_dir is shared for all languages.
         checkout_dir = settings.DOCS_BUILD_ROOT.joinpath("sources", release.version)
         parent_build_dir = settings.DOCS_BUILD_ROOT.joinpath(

docs/models.py

@@ -5,6 +5,7 @@
 from functools import partial, reduce
 from pathlib import Path
 
+import requests
 from django.conf import settings
 from django.contrib.postgres.indexes import GinIndex
 from django.contrib.postgres.search import (
@@ -34,8 +35,16 @@
     START_SEL,
     STOP_SEL,
     TSEARCH_CONFIG_LANGUAGES,
+    DocumentationCategory,
+    fetch_html,
     get_document_search_vector,
 )
+from .utils import extract_inner_html
+
+
+def get_search_config(lang):
+    """Determine the PostgreSQL search language"""
+    return TSEARCH_CONFIG_LANGUAGES.get(lang[:2], DEFAULT_TEXT_SEARCH_CONFIG)
 
 
 class DocumentReleaseQuerySet(models.QuerySet):
@@ -175,7 +184,7 @@ def sync_to_db(self, decoded_documents):
         the database. Deletes all the release's documents first then
         reinserts them as needed.
         """
-        self.documents.all().delete()
+        self.documents.exclude(metadata__parents=DocumentationCategory.WEBSITE).delete()
 
         # Read excluded paths from robots.docs.txt.
         robots_path = settings.BASE_DIR.joinpath(
@@ -206,16 +215,66 @@ def sync_to_db(self, decoded_documents):
                 path=document_path,
                 title=html.unescape(strip_tags(document["title"])),
                 metadata=document,
-                config=TSEARCH_CONFIG_LANGUAGES.get(
-                    self.lang[:2], DEFAULT_TEXT_SEARCH_CONFIG
-                ),
+                config=get_search_config(self.lang),
             )
-        for document in self.documents.all():
+        for document in self.documents.exclude(
+            metadata__parents=DocumentationCategory.WEBSITE
+        ):
             document.metadata["breadcrumbs"] = list(
                 Document.objects.breadcrumbs(document).values("title", "path")
             )
             document.save(update_fields=("metadata",))
 
+    def sync_from_sitemap(self, force=False):
+        from djangoproject.urls.www import sitemaps
+
+        if not self.is_dev:
+            return
+
+        if force:
+            Document.objects.filter(
+                metadata__parents=DocumentationCategory.WEBSITE
+            ).delete()
+
+        doc_urls = set(
+            Document.objects.filter(
+                metadata__parents=DocumentationCategory.WEBSITE
+            ).values_list("path", flat=True)
+        )
+
+        for sitemap in sitemaps.values():
+            for url in sitemap().get_urls():
+                path = url["location"]
+                if path in doc_urls:
+                    continue
+                try:
+                    page_html = fetch_html(path)
+                except requests.RequestException:
+                    continue
+                try:
+                    main_html = extract_inner_html(page_html, tag="main")
+                    title = extract_inner_html(page_html, tag="h1")
+                except ValueError:
+                    continue
+                Document.objects.create(
+                    release=self,
+                    path=path,
+                    title=title,
+                    metadata={
+                        "body": main_html,
+                        "breadcrumbs": [
+                            {
+                                "path": DocumentationCategory.WEBSITE,
+                                "title": "Website",
+                            },
+                        ],
+                        "parents": DocumentationCategory.WEBSITE,
+                        "title": title,
+                        "toc": "",
+                    },
+                    config=get_search_config(self.lang),
+                )
+
 
 def _clean_document_path(path):
     # We have to be a bit careful to reverse-engineer the correct
@@ -228,7 +287,9 @@ def _clean_document_path(path):
 
 
 def document_url(doc):
-    if doc.path:
+    if doc.metadata.get("parents") == DocumentationCategory.WEBSITE:
+        return doc.path
+    elif doc.path:
         kwargs = {
             "lang": doc.release.lang,
             "version": doc.release.version,
@@ -273,6 +334,14 @@ def search(self, query_text, release, document_category=None):
                 config=models.F("config"),
             )
             base_filter = Q(release_id=release.id)
+            if release.lang == "en" and not release.is_dev:
+                dev_release = DocumentRelease.objects.get_by_version_and_lang(
+                    "dev", "en"
+                )
+                base_filter |= Q(
+                    release_id=dev_release.id,
+                    metadata__parents=DocumentationCategory.WEBSITE,
+                )
             if document_category:
                 base_filter &= Q(metadata__parents__startswith=document_category)
             base_qs = (

docs/search.py

@@ -1,3 +1,4 @@
+import requests
 from django.contrib.postgres.search import SearchVector
 from django.db.models import TextChoices
 from django.db.models.fields.json import KeyTextTransform
@@ -67,10 +68,41 @@ class DocumentationCategory(TextChoices):
     TOPICS = "topics", _("Using Django")
     HOWTO = "howto", _("How-to guides")
     RELEASE_NOTES = "releases", _("Release notes")
+    WEBSITE = "website", _("Django Website")
 
     @classmethod
     def parse(cls, value, default=None):
         try:
             return cls(value)
         except ValueError:
             return None
+
+
+def fetch_html(url, timeout=10):
+    """
+    Fetch the HTML of a page if status code is 200.
+    Simulates a human browser and accepts only text/html.
+    """
+
+    headers = {
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0"
+        ),
+        "Accept": "text/html",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+
+    response = requests.get(url, headers=headers, timeout=timeout)
+
+    if response.status_code == 200:
+        content_type = response.headers.get("Content-Type", "")
+        if "text/html" in content_type:
+            return response.text
+        else:
+            raise requests.RequestException(f"Unexpected Content-Type: {content_type}")
+    else:
+        raise requests.RequestException(
+            f"Failed to fetch {url}, status code: {response.status_code}"
+        )

docs/templates/docs/search_results.html

@@ -43,11 +43,11 @@ <h2>{% translate "No search query given" %}</h2>
         {% for result in page.object_list %}
           <dt>
             <h2 class="result-title">
-              <a href="{% url 'document-detail' lang=result.release.lang version=result.release.version url=result.path host 'docs' %}{% if not start_sel in result.headline %}{{ result.highlight|fragment }}{% endif %}">{{ result.headline|safe }}</a>
+              <a href="{{ result.get_absolute_url }}{% if not start_sel in result.headline %}{{ result.highlight|fragment }}{% endif %}">{{ result.headline|safe }}</a>
             </h2>
             <span class="meta breadcrumbs">
               {% for breadcrumb in result.breadcrumbs %}
-                <a href="{% url 'document-detail' lang=result.release.lang version=result.release.version url=breadcrumb.path host 'docs' %}">{{ breadcrumb.title }}</a>{% if not forloop.last %} <span class="arrow">»</span>{% endif %}
+                <a href="{{ result.get_absolute_url }}">{{ breadcrumb.title }}</a>{% if not forloop.last %} <span class="arrow">»</span>{% endif %}
               {% endfor %}
             </span>
           </dt>
@@ -60,7 +60,7 @@ <h2 class="result-title">
               <ul class="code-links">
                 {% for name, value in result_code_links.items %}
                   <li>
-                    <a href="{% url 'document-detail' lang=result.release.lang version=result.release.version url=result.path host 'docs' %}#{{ value.full_path }}">
+                    <a href="{{ result.get_absolute_url }}#{{ value.full_path }}">
                       <div>
                         <code>{{ name }}</code>
                         {% if value.module_path %}<div class="meta">{{ value.module_path }}</div>{% endif %}