CIHelper: offer to update the mail-to-commit and commit-to-mail notes

This ports the Azure Pipeline "Update GitGitGadget's commit to mail
notes" (https://dev.azure.com/gitgitgadget/git/_build?definitionId=9)
logic to a method of the `CIHelper` class that will be used in a GitHub
Action so that the Azure Pipeline can be retired.

This was the only Azure Pipeline of GitGitGadget which did not,
actually, use the `misc-helper.ts` script, but instead was opaquely
implemented as a lengthy shell scriptlet inside the Pipeline definition
that called two of the shell scripts in the `gitgitgadget/gitgitgadget`
repository: `lookup-commit.sh` and `update-mail-to-commit-notes.sh`.

Since the scripts that are called by the new method expect a persisted,
non-partial clone of the Git mailing list, we need to play a couple of
games here to make it work in a GitHub workflow (that runs on ephemeral
runners where the repository has to be cloned afresh in every run).

Also: These shell scripts, by virtue of needing to be interpreted by a
shell interpreter outside of node.js, need to be copied into the
`dist/script/` subdirectory before publishing the GitHub Action, so that
they can be found and interpreted as expected.

Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>

Author: dscho

.vscode/settings.json

@@ -96,6 +96,7 @@
         "CIsmtphost",
         "CIsmtppass",
         "CIsmtpopts",
+        "GITGIT_(|DIR|GIT_REMOTE|MAIL_REMOTE|MAIL_EPOCH)",
         "mailrepo",
         "parsePRURLInput",
         "users\\.noreply\\.github\\.com",

lib/ci-helper.ts

@@ -2,10 +2,11 @@ import * as core from "@actions/core";
 import * as fs from "fs";
 import * as os from "os";
 import * as util from "util";
+import { spawnSync } from "child_process";
 import addressparser from "nodemailer/lib/addressparser/index.js";
 import path from "path";
 import { ILintError, LintCommit } from "./commit-lint.js";
-import { commitExists, git, emptyTreeName } from "./git.js";
+import { commitExists, git, emptyTreeName, revParse } from "./git.js";
 import { GitNotes } from "./git-notes.js";
 import { GitGitGadget, IGitGitGadgetOptions } from "./gitgitgadget.js";
 import { getConfig } from "./gitgitgadget-config.js";
@@ -70,6 +71,7 @@ export class CIHelper {
     public async setupGitHubAction(setupOptions?: {
         needsMailingListMirror?: boolean;
         needsUpstreamBranches?: boolean;
+        needsMailToCommitNotes?: boolean;
     }): Promise<void> {
         // help dugite realize where `git` is...
         const gitExecutable = os.type() === "Windows_NT" ? "git.exe" : "git";
@@ -133,14 +135,18 @@ export class CIHelper {
             await git(["config", key, value], { workDir: this.workDir });
         }
         console.time("fetch Git notes");
+        const notesRefs = [GitNotes.defaultNotesRef];
+        if (setupOptions?.needsMailToCommitNotes) {
+            notesRefs.push("refs/notes/mail-to-commit", "refs/notes/commit-to-mail");
+        }
         await git(
             [
                 "fetch",
                 "--filter=blob:limit=1g", // let's fetch the notes with all of their blobs
                 "--no-tags",
                 "origin",
                 "--depth=1",
-                `+${GitNotes.defaultNotesRef}:${GitNotes.defaultNotesRef}`,
+                ...notesRefs.map((ref) => `+${ref}:${ref}`),
             ],
             {
                 workDir: this.workDir,
@@ -217,7 +223,7 @@ export class CIHelper {
                     "remote.mirror.promisor=false", // let's fetch the mails with all of their contents
                     "fetch",
                     "mirror",
-                    "--depth=50",
+                    `--depth=${setupOptions?.needsMailToCommitNotes ? 5000 : 50}`,
                     "+REF:REF".replace("REF", this.config.mailrepo.mirrorRef || `refs/heads/lore-${epoch}`),
                 ],
                 {
@@ -317,6 +323,60 @@ export class CIHelper {
         await this.commit2mailNotes.appendCommitNote(gitGitCommit, messageId);
     }
 
+    /**
+     * Update the `commit-to-mail` and `mail-to-commit` Git notes refs.
+     */
+    public async updateMailToCommitNotes(): Promise<void> {
+        // We'll assume that the `commit-to-mail` and `mail-to-commit` notes refs are up to date
+        const commit2MailTipCommit = await revParse("refs/notes/commit-to-mail", this.workDir);
+        const dir = fileURLToPath(new URL(".", import.meta.url));
+        const lookupCommitScriptPath = path.resolve(dir, "..", "script", "lookup-commit.sh");
+        console.time("lookup-commit.sh");
+        const lookupCommitResult = spawnSync("sh", ["-x", lookupCommitScriptPath, "--notes", "update"], {
+            stdio: "inherit",
+            env: {
+                ...process.env,
+                GITGIT_DIR: this.workDir,
+                GITGIT_GIT_REMOTE: this.urlRepo,
+                LORE_GIT_DIR: this.mailingListMirror,
+                GITGIT_MAIL_REMOTE: this.config.mailrepo.url,
+                GITGIT_MAIL_EPOCH: "1",
+            },
+        });
+        console.timeEnd("lookup-commit.sh");
+        if (lookupCommitResult.status !== 0) throw new Error("lookup-commit.sh failed");
+        // If there were no updates, we are done
+        if (commit2MailTipCommit === (await revParse("refs/notes/commit-to-mail", this.workDir))) return;
+
+        const updateMailToCommitNotesScriptPath = path.resolve(dir, "..", "script", "update-mail-to-commit-notes.sh");
+        console.time("update-mail-to-commit-notes.sh");
+        const updateMailToCommitNotesResult = spawnSync("sh", ["-x", updateMailToCommitNotesScriptPath], {
+            stdio: "inherit",
+            env: {
+                ...process.env,
+                GITGIT_DIR: this.workDir,
+                GITGIT_GIT_REMOTE: this.urlRepo,
+            },
+        });
+        console.timeEnd("update-mail-to-commit-notes.sh");
+        if (updateMailToCommitNotesResult.status !== 0) throw new Error("update-mail-to-commit-notes.sh failed");
+
+        const mail2commitNotes = new GitNotes(this.workDir, "refs/notes/mail-to-commit");
+        await mail2commitNotes.push(this.urlRepo, this.notesPushToken);
+        const commit2MailNotes = new GitNotes(this.workDir, "refs/notes/commit-to-mail");
+        await commit2MailNotes.push(this.urlRepo, this.notesPushToken);
+
+        const commit2MailTipPatch = await git(["show", "refs/notes/commit-to-mail"], { workDir: this.workDir });
+        // Any unhandled commit will get annotated with "no match"
+        // To list all of them, the tip commit's diff is parsed and the commit hash is
+        // extracted from the "filename" on the `+++ b/` line.
+        const noMatch = commit2MailTipPatch
+            .split("\ndiff --git ")
+            .filter((d) => d.endsWith("+no match"))
+            .map((d) => d.split("\n+++ b/")[1].split("\n")[0].replace("/", ""));
+        if (noMatch.length) throw new Error(`Could not find mail(s) for: ${noMatch.join("\n")}`);
+    }
+
     /**
      * Given a Message-Id, identify the upstream commit (if any), and if there
      * is one, and if it was not yet recorded in GitGitGadget's metadata, record