feat(treewide): lazy load resources from internet on first use

Dockerfile

@@ -8,7 +8,6 @@ RUN apt-get -y update && \
 WORKDIR /home/node/app
 USER node
 COPY --chown=node:node . .
-# This also installs hsts and tld data files in a postinstall script:
 RUN npm install
 
 ARG GIT_SHA=dev
@@ -18,6 +17,5 @@ RUN env
 
 ENV RUN_ID=${RUN_ID}
 ENV GIT_SHA=${GIT_SHA}
-ENV NODE_EXTRA_CA_CERTS=node_modules/extra_certs/ca_bundle/ca_intermediate_bundle.pem
 EXPOSE 8080
 CMD [ "node", "src/api/index.js" ]

bin/wrapper.js

@@ -9,13 +9,6 @@ import { fileURLToPath } from "node:url";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
-// Set the environment variable for extra CA certificates
-let caCertPath = import.meta.resolve("node_extra_ca_certs_mozilla_bundle");
-caCertPath = new URL(caCertPath).pathname;
-caCertPath = path.dirname(caCertPath);
-caCertPath = path.join(caCertPath, "ca_bundle", "ca_intermediate_bundle.pem");
-process.env.NODE_EXTRA_CA_CERTS = caCertPath;
-
 // The target script you want to run (relative to this script's directory)
 const targetScript = path.join(__dirname, "..", "src", "scan.js");
 

package.json

@@ -9,16 +9,14 @@
     "npm": ">=9.0.0"
   },
   "scripts": {
-    "start": "NODE_EXTRA_CA_CERTS=node_modules/node_extra_ca_certs_mozilla_bundle/ca_bundle/ca_intermediate_root_bundle.pem node src/api/index.js",
-    "dev": "NODE_EXTRA_CA_CERTS=node_modules/node_extra_ca_certs_mozilla_bundle/ca_bundle/ca_intermediate_root_bundle.pem nodemon src/api/index.js",
+    "start": "node src/api/index.js",
+    "dev": "nodemon src/api/index.js",
     "test": "CONFIG_FILE=conf/config-test.json mocha",
     "tsc": "tsc -p jsconfig.json",
-    "updateHsts": "node src/retrieve-hsts.js",
-    "updateTldList": "node src/retrieve-tld-list.js",
     "refreshMaterializedViews": "node src/maintenance/index.js",
+    "refreshCache": "node src/cache.js",
     "maintenance": "node src/maintenance/index.js",
-    "migrate": "node -e 'import(\"./src/database/migrate.js\").then( m => m.migrateDatabase() )'",
-    "postinstall": "npm run updateHsts && npm run updateTldList"
+    "migrate": "node -e 'import(\"./src/database/migrate.js\").then( m => m.migrateDatabase() )'"
   },
   "bin": {
     "mdn-http-observatory-scan": "bin/wrapper.js"
@@ -59,7 +57,7 @@
     "http-cookie-agent": "^7.0.1",
     "ip": "^2.0.1",
     "jsdom": "^27.0.0",
-    "node_extra_ca_certs_mozilla_bundle": "^1.0.7",
+    "papaparse": "^5.5.3",
     "pg": "^8.16.2",
     "pg-format": "^1.0.4",
     "pg-native": "^3.5.2",

src/analyzer/hsts.js

@@ -1,9 +1,6 @@
 import fs from "fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
 import { Site } from "../site.js";
-
-const dirname = path.dirname(fileURLToPath(import.meta.url));
+import { HSTS_PRELOAD_PATH } from "../cache.js";
 
 /**
  * @type {import("../types.js").Hsts | null}
@@ -15,15 +12,8 @@ let hstsMap = null;
  */
 export function hsts() {
   if (!hstsMap) {
-    const filePath = path.join(
-      dirname,
-      "..",
-      "..",
-      "conf",
-      "hsts-preload.json"
-    );
     hstsMap = new Map(
-      Object.entries(JSON.parse(fs.readFileSync(filePath, "utf8")))
+      Object.entries(JSON.parse(fs.readFileSync(HSTS_PRELOAD_PATH, "utf8")))
     );
   }
   return hstsMap;

src/api/index.js

@@ -1,7 +1,10 @@
 import { CONFIG } from "../config.js";
 import { createServer } from "./server.js";
+import { setupCache } from "../cache.js";
 
 async function main() {
+  await setupCache();
+
   const server = await createServer();
   try {
     await server.listen({

src/api/v2/utils.js

@@ -1,8 +1,6 @@
 import ip from "ip";
 import dns from "node:dns";
 import fs from "fs";
-import { fileURLToPath } from "node:url";
-import path from "node:path";
 import {
   InvalidHostNameError,
   InvalidHostNameIpError,
@@ -28,6 +26,7 @@ import { PolicyResponse } from "./schemas.js";
 import { Expectation } from "../../types.js";
 import { TEST_TITLES } from "../../grader/charts.js";
 import { scan } from "../../scanner/index.js";
+import { TLD_LIST_PATH } from "../../cache.js";
 
 /**
  *
@@ -50,23 +49,14 @@ export function isIp(hostname) {
  * @type {Set<string> | null}
  */
 let tldSet = null;
-const dirname = path.dirname(fileURLToPath(import.meta.url));
 
 /**
  * Get the cached set of top level domains.
  * @returns {Set<string>}
  */
 function tlds() {
   if (!tldSet) {
-    const filePath = path.join(
-      dirname,
-      "..",
-      "..",
-      "..",
-      "conf",
-      "tld-list.json"
-    );
-    tldSet = new Set(JSON.parse(fs.readFileSync(filePath, "utf8")));
+    tldSet = new Set(JSON.parse(fs.readFileSync(TLD_LIST_PATH, "utf8")));
   }
   return tldSet;
 }

src/ca-bundle.js

@@ -0,0 +1,72 @@
+import axios from "axios";
+import { writeFile } from "fs/promises";
+import Papa from "papaparse";
+
+const INTERMEDIATE_CA_URL =
+  "https://ccadb.my.salesforce-sites.com/mozilla/PublicAllIntermediateCertsWithPEMCSV";
+
+const ROOT_CA_URL =
+  "https://ccadb.my.salesforce-sites.com/mozilla/IncludedCACertificateReportPEMCSV";
+
+/**
+ * @param {string} url
+ * @returns {Promise<string[]>}
+ */
+async function downloadCertificates(url) {
+  let r;
+  try {
+    r = await axios.get(url);
+  } catch (error) {
+    throw Error(`Failed to get data: ${error}`);
+  }
+
+  const data = Papa.parse(r.data, { header: true }).data;
+  const output = [];
+  for (const entry of data) {
+    // Remove quotes from beginning and end of certificate
+    const certPem = entry["PEM Info"].slice(1, -1);
+    const commonName = entry["Common Name or Certificate Name"];
+    output.push(`${commonName}\n${certPem}`);
+  }
+  return output;
+}
+
+/**
+ * @returns {Promise<string>}
+ */
+async function retrieveCABundle() {
+  // Download at the same time
+  const values = await Promise.all([
+    downloadCertificates(INTERMEDIATE_CA_URL),
+    downloadCertificates(ROOT_CA_URL),
+  ]);
+
+  const intermediateCACerts = values[0];
+  const rootCACerts = values[1];
+
+  const combinedCACerts = intermediateCACerts.concat(rootCACerts);
+  return combinedCACerts.join("\n\n");
+}
+
+/**
+ * @param {string} filePath
+ */
+export async function retrieveAndStoreCABundle(filePath) {
+  const caBundle = await retrieveCABundle();
+
+  try {
+    await writeFile(filePath, caBundle);
+    console.log(`Downloaded Mozilla CA bundle and saved it to ${filePath}`);
+  } catch (error) {
+    console.error("Error writing file:", error);
+    return;
+  }
+}
+
+/**
+ *
+ * @param {string} filePath
+ */
+export async function setupCABundle(filePath) {
+  process.env.NODE_EXTRA_CA_CERTS = filePath;
+}