Store and serve windows binary downloads for releases (boostorg#1797)

Author: gregjkal

templates/versions/detail.html

@@ -7,7 +7,7 @@
 {% block title %}{% blocktrans with version_name=version.display_name %}Boost {{ version_name }}{% endblocktrans %}{% endblock %}
 {% block description %}{% blocktrans with version_name=version.display_name %}Discover what's new in Boost {{ version_name }}{% endblocktrans %}{% endblock %}
 {% block content %}
-  <main class="content">
+  <main>
     {% if selected_version %}
     <div class="py-3 px-3 md:mt-3 md:px-0 mb-0 w-full flex flex-row flex-nowrap items-center"
             x-data="{'showSearch': false}"
@@ -29,26 +29,61 @@
 
     <section class="content">
       <div class="pb-2 w-full h-auto md:pb-0 md:w-auto">
-        <div class="flex flex-col">
+        <div class="flex flex-col h-full max-w-md">
           <div class="h-8">
             <span class="block pb-1 text-xs md:text-base font-bold">{{ version.release_date|date:"F j, Y" }}</span>
           </div>
           <div class="-ml-2 h-3"></div>
-          <div class="-ml-2 h-14">
-            <a class="block items-center py-1 px-2 rounded cursor-pointer hover:bg-gray-100 dark:hover:bg-slate text-sky-600 dark:text-sky-300 hover:text-orange dark:hover:text-orange"
-               href="{{ documentation_url }}">
-              <span class="dark:text-white text-slate">Documentation</span>
-              <span class="block text-xs">{{ request.scheme }}://{{ request.get_host }}{{ documentation_url }}</span>
-            </a>
-          </div>
-          <div class="-ml-2 h-14">
-            <a class="block items-center py-1 px-2 rounded cursor-pointer hover:bg-gray-100 dark:hover:bg-slate text-sky-600 dark:text-sky-300 hover:text-orange dark:hover:text-orange"
-               href="{{ version.github_url }}">
-              <i class="float-right mt-1 fab fa-github"></i>
-              <span class="dark:text-white text-slate">Source Code</span>
-              <span class="block text-xs">{{ version.github_url|cut:"https://" }}</span>
-            </a>
-          </div>
+          <div class="flex flex-col h-full justify-between">
+            <div>
+              <div class="-ml-2 h-14">
+                <a class="block items-center py-1 px-2 rounded cursor-pointer hover:bg-gray-100 dark:hover:bg-slate text-sky-600 dark:text-sky-300 hover:text-orange dark:hover:text-orange"
+                   href="{{ documentation_url }}">
+                  <span class="dark:text-white text-slate">Documentation</span>
+                  <span class="block text-xs">{{ request.scheme }}://{{ request.get_host }}{{ documentation_url }}</span>
+                </a>
+              </div>
+              <div class="-ml-2 h-14">
+                <a class="block items-center py-1 px-2 rounded cursor-pointer hover:bg-gray-100 dark:hover:bg-slate text-sky-600 dark:text-sky-300 hover:text-orange dark:hover:text-orange"
+                   href="{{ version.github_url }}">
+                  <i class="float-right mt-1 fab fa-github"></i>
+                  <span class="dark:text-white text-slate">Source Code</span>
+                  <span class="block text-xs">{{ version.github_url|cut:"https://" }}</span>
+                </a>
+              </div>
+            </div>
+            {% if not version.beta %}
+              {% if deps.added or deps.removed %}
+              <div class="-ml-2">
+                <div class="block items-center py-1 px-2">
+                  <span class="dark:text-white text-slate font-semibold">Dependencies</span>
+                  <div class="text-base whitespace-normal">
+                    {% if deps.added %}
+                      There {{ deps.added|pluralize:"was,were" }}
+                      <span class="text-red-700">
+                        {{ deps.added }} dependenc{{ deps.added|pluralize:"y,ies"}} added
+                      </span>
+                      (in {{ deps.increased_dep_lib_count }} librar{{ deps.increased_dep_lib_count|pluralize:"y,ies" }})
+                    {% endif %}
+                    {% if deps.added and deps.removed %}
+                      and
+                    {% endif %}
+                    {% if deps.removed %}
+                      {% if not deps.added %}
+                        There {{ deps.removed|pluralize:"was,were" }}
+                      {% endif %}
+                      <span class="text-[rgb(14,174,96)] dark:text-green">
+                        {{ deps.removed }} dependenc{{ deps.removed|pluralize:"y,ies"}} removed
+                      </span>
+                      (in {{ deps.decreased_dep_lib_count }} librar{{ deps.decreased_dep_lib_count|pluralize:"y,ies" }})
+                    {% endif %}
+                    this release.
+                  </div>
+                </div>
+              </div>
+            </div>
+          {% endif %}
+        {% endif %}
 
         </div>
       </div>
@@ -80,7 +115,7 @@
                       {% if forloop.first %}
                         <th scope="row"
                             rowspan="{{ download_files|length }}"
-                            class="p-2 h-14 whitespace-nowrap border border-r-0 {% if not forloop.parentloop.last %}border-b-0 {% endif %}border-gray-400 dark:border-slate dark:bg-charcoal text-center">
+                            class="p-2 h-14 whitespace-normal border border-r-0 {% if not forloop.parentloop.last %}border-b-0 {% endif %}border-gray-400 dark:border-slate dark:bg-charcoal text-center">
                           <i class="fab fa-{% if os == 'Unix' %}linux{% else %}windows{% endif %}"></i> {{ os }}
                         </th>
                       {% endif %}
@@ -96,7 +131,7 @@
                           </svg>
                         </button>
                       </td>
-                      <td class="border pr-2 {% if not forloop.last or not forloop.parentloop.last %}border-b-0 {% endif %}border-l-0 border-gray-400 dark:border-slate truncCell dark:bg-charcoal"
+                      <td class="border pr-2 text-xs {% if not forloop.last or not forloop.parentloop.last %}border-b-0 {% endif %}border-l-0 border-gray-400 dark:border-slate truncCell dark:bg-charcoal"
                           title="{{ download.checksum }}">
                         <span class="hidden xl:block">{{ download.checksum }}</span>
                         <span class="hidden md:block xl:hidden">{{ download.checksum|truncate_middle:20 }}</span>
@@ -120,41 +155,6 @@
       </section>
     {% endif %}
 
-    {% if not version.beta %}
-      {% if deps.added or deps.removed %}
-        <section id="dependencyChanges"
-          class="p-6 my-4 bg-white md:rounded-lg md:shadow-lg dark:text-white text-slate dark:bg-charcoal dark:bg-neutral-700">
-          <h2 class="text-2xl mt-0">Dependencies</h2>
-          <div>
-            {% if deps.added %}
-              There {{ deps.added|pluralize:"was,were" }}
-              <span class="text-red-700">
-                {{ deps.added }} dependenc{{ deps.added|pluralize:"y,ies"}} added
-              </span>
-              <span>
-                (in {{ deps.increased_dep_lib_count }} librar{{ deps.increased_dep_lib_count|pluralize:"y,ies" }})
-              </span>
-            {% endif %}
-            {% if deps.added and deps.removed %}
-              and
-            {% endif %}
-            {% if deps.removed %}
-              {% if not deps.added %}
-                There {{ deps.removed|pluralize:"was,were" }}
-              {% endif %}
-              <span class="text-[rgb(14,174,96)] dark:text-green">
-                {{ deps.removed }} dependenc{{ deps.removed|pluralize:"y,ies"}} removed
-              </span>
-              <span>
-                (in {{ deps.decreased_dep_lib_count }} librar{{ deps.decreased_dep_lib_count|pluralize:"y,ies" }})
-              </span>
-            {% endif %}
-            this release.
-          </div>
-        </section>
-      {% endif %}
-    {% endif %}
-
     {% if top_contributors_release %}
       <section id="releaseContributors"
         class="p-6 my-4 bg-white md:rounded-lg md:shadow-lg dark:text-white text-slate dark:bg-charcoal dark:bg-neutral-700">

versions/management/commands/import_archives_release_data.py

@@ -9,6 +9,9 @@
     get_archives_download_data,
     get_archives_download_uris_for_release,
     store_release_downloads_for_version,
+    get_binaries_download_data,
+    get_binary_checksums,
+    get_binary_download_uris_for_release,
 )
 
 
@@ -32,18 +35,26 @@ def command(release):
     for v in versions:
         version_num = v.name.replace("boost-", "")
         try:
-            archives_data = get_archives_download_uris_for_release(version_num)
+            archives_urls = get_archives_download_uris_for_release(version_num)
+            binaries_urls = get_binary_download_uris_for_release(version_num)
+            file_urls = archives_urls + binaries_urls
         except requests.exceptions.HTTPError:
             print(f"Skipping {version_num}, error retrieving release data")
             continue
 
         download_data = []
-        for d in archives_data:
+        checksums = dict()
+        for url in file_urls:
             try:
-                data = get_archives_download_data(d)
+                if "/binaries/" in url:
+                    if not checksums:
+                        checksums = get_binary_checksums(url)
+                    data = get_binaries_download_data(url, checksums)
+                else:
+                    data = get_archives_download_data(url)
                 download_data.append(data)
             except (requests.exceptions.HTTPError, ValueError):
-                print(f"Skipping {d}, error retrieving download data")
+                print(f"Skipping {url}; error retrieving download data")
                 continue
 
             store_release_downloads_for_version(v, download_data)

versions/releases.py

@@ -20,23 +20,17 @@
 session = requests.Session()
 
 
-def get_archives_download_uris_for_release(release: str = "1.81.0") -> list:
-    """Get the download information for a Boost release from the Boost Archives.
-
-    Args:
-        release (str): The Boost release to get download information for. Defaults to
-            "1.81.0".
-
-    Returns:
-        list: A list of URLs to download the release data from.
-    """
-    file_extensions = [".tar.bz2", ".tar.gz", ".7z", ".zip"]
-    file_name_excludes = ["_rc"]
-
-    if "beta" in release:
-        release_path = f"{settings.ARCHIVES_URL}beta/{release}/source/"
-    else:
-        release_path = f"{settings.ARCHIVES_URL}release/{release}/source/"
+def get_download_uris_for_release(
+    release: str,
+    subdir: str,
+    file_extensions: list[str],
+    file_name_excludes: list[str] = None,
+) -> list[str]:
+    """Get the download URIs for a Boost release from the Boost Archives."""
+    file_name_excludes = file_name_excludes or []
+
+    release_type = "beta" if "beta" in release else "release"
+    release_path = f"{settings.ARCHIVES_URL}{release_type}/{release}/{subdir}/"
 
     try:
         resp = session.get(release_path)
@@ -47,18 +41,32 @@ def get_archives_download_uris_for_release(release: str = "1.81.0") -> list:
         )
         raise
 
-    # Get the list of archives downloads for this release
     soup = BeautifulSoup(resp.text, "html.parser")
-    uris = []
-    for a in soup.find_all("a"):
-        uri = a.get("href")
-        # Only include the download links with valid file extensions.
-        if any(uri.endswith(ext) for ext in file_extensions):
-            # Exclude release candidates
-            if not any(exclude in uri for exclude in file_name_excludes):
-                uris.append(f"{release_path}{uri}")
+    return [
+        f"{release_path}{a.get('href')}"
+        for a in soup.find_all("a")
+        if a.get("href")
+        and any(a.get("href").endswith(ext) for ext in file_extensions)
+        and not any(exclude in a.get("href") for exclude in file_name_excludes)
+    ]
+
+
+def get_archives_download_uris_for_release(release: str = "1.81.0") -> list[str]:
+    return get_download_uris_for_release(
+        release=release,
+        subdir="source",
+        file_extensions=[".tar.bz2", ".tar.gz", ".7z", ".zip"],
+        file_name_excludes=["_rc"],
+    )
 
-    return uris
+
+def get_binary_download_uris_for_release(release: str) -> list[str]:
+    return get_download_uris_for_release(
+        release=release,
+        subdir="binaries",
+        file_extensions=[".exe", ".7z"],
+        file_name_excludes=["_rc"],
+    )
 
 
 def get_artifactory_download_uris_for_release(release: str = "1.81.0") -> list:
@@ -145,6 +153,33 @@ def get_archives_download_data(url):
     }
 
 
+def get_binaries_download_data(url: str, checksums: dict) -> dict:
+    filename = url.split("/")[-1]
+    return {
+        "url": url,
+        "operating_system": "Windows (Bin)",
+        "checksum": checksums[filename],
+        "display_name": filename,
+    }
+
+
+def get_binary_checksums(url: str) -> dict:
+    binaries_url = "/".join(url.split("/")[:-1])
+    checksum_url = binaries_url + "/SHA256SUMS"
+    try:
+        resp = session.get(checksum_url)
+        resp.raise_for_status()
+    except requests.exceptions.HTTPError as e:
+        logger.error("get_binary_checksums", exc_msg=str(e), url=checksum_url)
+        raise
+
+    checksums = {}
+    for line in resp.text.strip().splitlines():
+        checksum, filename = line.split(maxsplit=1)
+        checksums[filename] = checksum
+    return checksums
+
+
 def get_artifactory_download_data(url):
     """Get the download information for a Boost release from the Boost artifactory."""
     try:

versions/views.py

@@ -51,7 +51,10 @@ def get_context_data(self, **kwargs):
             context["is_current_release"] = False
             return context
 
-        downloads = obj.downloads.all().order_by("operating_system")
+        downloads = obj.downloads.all().order_by("operating_system", "display_name")
+        for dl in downloads:
+            dl.operating_system = dl.operating_system.replace("Bin", "Binary")
+
         context["downloads"] = {
             k: list(v)
             for k, v in groupby(downloads, key=attrgetter("operating_system"))