feat(treewide): lazy load resources from internet on first use

Now, nothing is dynamically downloaded from the internet at install
time. Instead, on first use, a cache direcory is created
(`~/.cache/mdn-http-observatory`) and these files downloaded:

- Mozilla CA Bundle
- HSTS preload list
- TLD list

When one of these files is not present, it is downloaded on the next
invocation. When it is present on the filesystem it is used right away.

"Refreshing" these resources can then simply be done by removing the
cache directory and restarting the application. It can also be done by
calling `npm run refreshCache` or via `npm run maintenance`.

This enables packaging mdn-http-observatory for Linux distros.
Additionaly, it makes the package a little easier to understand because
there is no loading of resources at various points at install time.

Author: nikstur

Dockerfile

@@ -18,6 +18,5 @@ RUN env
 
 ENV RUN_ID=${RUN_ID}
 ENV GIT_SHA=${GIT_SHA}
-ENV NODE_EXTRA_CA_CERTS=node_modules/extra_certs/ca_bundle/ca_intermediate_bundle.pem
 EXPOSE 8080
 CMD [ "node", "src/api/index.js" ]

bin/wrapper.js

@@ -9,13 +9,6 @@ import { fileURLToPath } from "node:url";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
-// Set the environment variable for extra CA certificates
-let caCertPath = import.meta.resolve("node_extra_ca_certs_mozilla_bundle");
-caCertPath = new URL(caCertPath).pathname;
-caCertPath = path.dirname(caCertPath);
-caCertPath = path.join(caCertPath, "ca_bundle", "ca_intermediate_bundle.pem");
-process.env.NODE_EXTRA_CA_CERTS = caCertPath;
-
 // The target script you want to run (relative to this script's directory)
 const targetScript = path.join(__dirname, "..", "src", "scan.js");
 

package-lock.json

@@ -9,16 +9,14 @@
     "npm": ">=9.0.0"
   },
   "scripts": {
-    "start": "NODE_EXTRA_CA_CERTS=node_modules/node_extra_ca_certs_mozilla_bundle/ca_bundle/ca_intermediate_root_bundle.pem node src/api/index.js",
-    "dev": "NODE_EXTRA_CA_CERTS=node_modules/node_extra_ca_certs_mozilla_bundle/ca_bundle/ca_intermediate_root_bundle.pem nodemon src/api/index.js",
+    "start": "node src/api/index.js",
+    "dev": "nodemon src/api/index.js",
     "test": "CONFIG_FILE=conf/config-test.json mocha",
     "tsc": "tsc -p jsconfig.json",
-    "updateHsts": "node src/retrieve-hsts.js",
-    "updateTldList": "node src/retrieve-tld-list.js",
     "refreshMaterializedViews": "node src/maintenance/index.js",
+    "refreshCache": "node src/cache.js",
     "maintenance": "node src/maintenance/index.js",
-    "migrate": "node -e 'import(\"./src/database/migrate.js\").then( m => m.migrateDatabase() )'",
-    "postinstall": "npm run updateHsts && npm run updateTldList"
+    "migrate": "node -e 'import(\"./src/database/migrate.js\").then( m => m.migrateDatabase() )'"
   },
   "bin": {
     "mdn-http-observatory-scan": "bin/wrapper.js"
@@ -59,7 +57,7 @@
     "http-cookie-agent": "^7.0.1",
     "ip": "^2.0.1",
     "jsdom": "^27.0.0",
-    "node_extra_ca_certs_mozilla_bundle": "^1.0.7",
+    "papaparse": "^5.5.3",
     "pg": "^8.16.2",
     "pg-format": "^1.0.4",
     "pg-native": "^3.5.2",

package.json

@@ -1,9 +1,6 @@
 import fs from "fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
 import { Site } from "../site.js";
-
-const dirname = path.dirname(fileURLToPath(import.meta.url));
+import { HSTS_PRELOAD_PATH } from "../cache.js";
 
 /**
  * @type {import("../types.js").Hsts | null}
@@ -15,15 +12,8 @@ let hstsMap = null;
  */
 export function hsts() {
   if (!hstsMap) {
-    const filePath = path.join(
-      dirname,
-      "..",
-      "..",
-      "conf",
-      "hsts-preload.json"
-    );
     hstsMap = new Map(
-      Object.entries(JSON.parse(fs.readFileSync(filePath, "utf8")))
+      Object.entries(JSON.parse(fs.readFileSync(HSTS_PRELOAD_PATH, "utf8")))
     );
   }
   return hstsMap;

src/analyzer/hsts.js

@@ -1,7 +1,10 @@
 import { CONFIG } from "../config.js";
 import { createServer } from "./server.js";
+import { setupCache } from "../cache.js";
 
 async function main() {
+  await setupCache();
+
   const server = await createServer();
   try {
     await server.listen({

src/api/index.js

@@ -1,8 +1,6 @@
 import ip from "ip";
 import dns from "node:dns";
 import fs from "fs";
-import { fileURLToPath } from "node:url";
-import path from "node:path";
 import {
   InvalidHostNameError,
   InvalidHostNameIpError,
@@ -28,6 +26,7 @@ import { PolicyResponse } from "./schemas.js";
 import { Expectation } from "../../types.js";
 import { TEST_TITLES } from "../../grader/charts.js";
 import { scan } from "../../scanner/index.js";
+import { TLD_LIST_PATH } from "../../cache.js";
 
 /**
  *
@@ -50,23 +49,14 @@ export function isIp(hostname) {
  * @type {Set<string> | null}
  */
 let tldSet = null;
-const dirname = path.dirname(fileURLToPath(import.meta.url));
 
 /**
  * Get the cached set of top level domains.
  * @returns {Set<string>}
  */
 function tlds() {
   if (!tldSet) {
-    const filePath = path.join(
-      dirname,
-      "..",
-      "..",
-      "..",
-      "conf",
-      "tld-list.json"
-    );
-    tldSet = new Set(JSON.parse(fs.readFileSync(filePath, "utf8")));
+    tldSet = new Set(JSON.parse(fs.readFileSync(TLD_LIST_PATH, "utf8")));
   }
   return tldSet;
 }

src/api/v2/utils.js

@@ -0,0 +1,72 @@
+import axios from "axios";
+import { writeFile } from "fs/promises";
+import Papa from "papaparse";
+
+const INTERMEDIATE_CA_URL =
+  "https://ccadb.my.salesforce-sites.com/mozilla/PublicAllIntermediateCertsWithPEMCSV";
+
+const ROOT_CA_URL =
+  "https://ccadb.my.salesforce-sites.com/mozilla/IncludedCACertificateReportPEMCSV";
+
+/**
+ * @param {string} url
+ * @returns {Promise<string[]>}
+ */
+async function downloadCertificates(url) {
+  let r;
+  try {
+    r = await axios.get(url);
+  } catch (error) {
+    throw Error(`Failed to get data: ${error}`);
+  }
+
+  const data = Papa.parse(r.data, { header: true }).data;
+  const output = [];
+  for (const entry of data) {
+    // Remove quotes from beginning and end of certificate
+    const certPem = entry["PEM Info"].slice(1, -1);
+    const commonName = entry["Common Name or Certificate Name"];
+    output.push(`${commonName}\n${certPem}`);
+  }
+  return output;
+}
+
+/**
+ * @returns {Promise<string>}
+ */
+async function retrieveCABundle() {
+  // Download at the same time
+  const values = await Promise.all([
+    downloadCertificates(INTERMEDIATE_CA_URL),
+    downloadCertificates(ROOT_CA_URL),
+  ]);
+
+  const intermediateCACerts = values[0];
+  const rootCACerts = values[1];
+
+  const combinedCACerts = intermediateCACerts.concat(rootCACerts);
+  return combinedCACerts.join("\n\n");
+}
+
+/**
+ * @param {string} filePath
+ */
+export async function retrieveAndStoreCABundle(filePath) {
+  const caBundle = await retrieveCABundle();
+
+  try {
+    await writeFile(filePath, caBundle);
+    console.log(`Downloaded Mozilla CA bundle and saved it to ${filePath}`);
+  } catch (error) {
+    console.error("Error writing file:", error);
+    return;
+  }
+}
+
+/**
+ *
+ * @param {string} filePath
+ */
+export async function setupCABundle(filePath) {
+  process.env.NODE_EXTRA_CA_CERTS = filePath;
+}