project runner: start/config of mutagen in sidecar

Author: williamstein

src/packages/file-server/ssh/auth.ts

@@ -69,8 +69,7 @@ export async function init({
         authorizedKeys,
       };
 
-      // HIGHLY SENSITVE!!!
-      // console.log("sending", resp);
+      //console.log("USING", { ...resp, privateKey: "xxx" });
 
       res.json(resp);
     } catch (err) {

src/packages/file-server/ssh/container.ts

@@ -50,10 +50,6 @@ export const start = reuseInFlight(
   }): Promise<{ sshPort: number }> => {
     let child = children[volume];
     if (child != null && child.exitCode == null) {
-      console.log("already running", {
-        publicKey,
-        curPublicKey: child.publicKey,
-      });
       // already running
       if (
         child.publicKey != publicKey ||
@@ -128,14 +124,13 @@ export const start = reuseInFlight(
             return true;
           }
           const ports = await getPorts({ volume });
-          console.log("got ports", ports);
           if (ports[22]) {
             // @ts-ignore
             child.sshPort = ports[22];
             return true;
           }
-        } catch {
-          console.log("got ports error");
+        } catch (err) {
+          console.log("got ports error", err);
         }
         return false;
       },

src/packages/project-runner/run/env.ts

@@ -91,5 +91,6 @@ export async function getEnvironment({
     CONAT_SERVER: conatServer.replace("localhost", "host.containers.internal"),
     COCALC_SECRET_TOKEN: secretTokenPath(HOME),
     BASE_PATH: base_path,
+    DEBIAN_FRONTEND: "noninteractive",
   };
 }

src/packages/project-runner/run/overlay.ts

@@ -93,7 +93,7 @@ function getMergedPath(project_id) {
   return join(PROJECT_ROOTS, project_id);
 }
 
-function getPaths({ home, image, project_id }) {
+export function getPaths({ home, image, project_id }) {
   const userOverlays = join(home, PROJECT_IMAGE_PATH, image);
   const upperdir = join(userOverlays, "upperdir");
   const workdir = join(userOverlays, "workdir");

src/packages/project-runner/run/podman.ts

@@ -20,7 +20,7 @@ import { nodePath } from "./mounts";
 import { isValidUUID } from "@cocalc/util/misc";
 import { ensureConfFilesExists, setupDataPath, writeSecretToken } from "./util";
 import { getEnvironment } from "./env";
-import { mkdir, readFile } from "node:fs/promises";
+import { mkdir, readFile, rm } from "node:fs/promises";
 import { spawn } from "node:child_process";
 import { getCoCalcMounts, COCALC_SRC } from "./mounts";
 import { setQuota } from "./filesystem";
@@ -36,8 +36,8 @@ import {
   type SshServersFunction,
   type LocalPathFunction,
 } from "@cocalc/conat/project/runner/types";
-import { write as writeMutagenConfig } from "@cocalc/backend/mutagen/config";
 import { initSshKeys } from "@cocalc/backend/mutagen/ssh-keys";
+import { PROJECT_IMAGE_PATH } from "@cocalc/util/db-schema/defaults";
 
 const logger = getLogger("project-runner:podman");
 const children: { [project_id: string]: any } = {};
@@ -78,15 +78,24 @@ export async function start({
   logger.debug("start: got home", { project_id, home });
   const mounts = getCoCalcMounts();
   const image = getImage(config);
-  await initSshKeys({ home, sshServers: await sshServers?.({ project_id }) });
+  const servers = await sshServers?.({ project_id });
+  await initSshKeys({ home, sshServers: servers });
 
   const env = await getEnvironment({
     project_id,
     env: config?.env,
     HOME: "/root",
     image,
   });
-  await startSidecar({ project_id, home, mounts, env, pod });
+  const initMutagen = await startSidecar({
+    image,
+    project_id,
+    home,
+    mounts,
+    env,
+    pod,
+    servers,
+  });
 
   const rootfs = await rootFilesystem.mount({ project_id, home, config });
   logger.debug("start: got rootfs", { project_id, rootfs });
@@ -95,11 +104,11 @@ export async function start({
   await ensureConfFilesExists(home);
   logger.debug("start: created conf files", { project_id });
 
-  await writeMutagenConfig({
-    home,
-    sync: config?.sync,
-    forward: config?.forward,
-  });
+  //   await writeMutagenConfig({
+  //     home,
+  //     sync: config?.sync,
+  //     forward: config?.forward,
+  //   });
 
   await setupDataPath(home);
   logger.debug("start: setup data path", { project_id });
@@ -119,6 +128,7 @@ export async function start({
   const args: string[] = [];
   args.push("run");
   args.push("--rm");
+  args.push("--replace");
   args.push("--user=0:0");
   args.push("--pod", pod);
 
@@ -157,16 +167,27 @@ export async function start({
   child.stderr.on("data", (chunk: Buffer) => {
     logger.debug(`project_id=${project_id}.stderr: `, chunk.toString());
   });
+
+  await initMutagen?.();
 }
 
-async function startSidecar({ project_id, mounts, env, pod, home }) {
+async function startSidecar({
+  image,
+  project_id,
+  mounts,
+  env,
+  pod,
+  home,
+  servers,
+}) {
   // sidecar: refactor
   const sidecarPodName = `sidecar-${project_id}`;
   const args2 = [
     "run",
     `--name=${sidecarPodName}`,
     "--detach",
     "--rm",
+    "--replace",
     "--pod",
     pod,
     "--init",
@@ -178,22 +199,72 @@ async function startSidecar({ project_id, mounts, env, pod, home }) {
   for (const name in env) {
     args2.push("-e", `${name}=${env[name]}`);
   }
+
   args2.push(sidecarImageName, "mutagen", "daemon", "run");
+
+  // always start with fresh .mutagen
+  await rm(join(home, ".mutagen-dev"), { force: true, recursive: true });
+
   await podman(args2);
+
+  if (servers.length == 0) {
+    return;
+  }
+
+  const upperdir = join(PROJECT_IMAGE_PATH, image, "upperdir");
   try {
+    await podman(
+      [
+        "exec",
+        sidecarPodName,
+        "rsync",
+        "--relative",
+        "-axH",
+        `${servers[0].name}:${upperdir}/`,
+        "/root/",
+      ],
+      10 * 60,
+    );
+  } catch (err) {
+    console.log(err);
+  }
+
+  return async () => {
     await podman([
       "exec",
       sidecarPodName,
-      "rsync",
-      "-axH",
-      "--exclude=.mutagen",
-      "--delete",
-      "file-server:/root/",
-      "/root/",
+      "mutagen",
+      "sync",
+      "create",
+      "--name=upperdir",
+      "--mode=one-way-replica",
+      "--symlink-mode=posix-raw",
+      join("/root", upperdir),
+      `${servers[0].name}:${upperdir}`,
     ]);
-  } catch (err) {
-    console.log(err);
-  }
+
+    await podman([
+      "exec",
+      sidecarPodName,
+      "mutagen",
+      "sync",
+      "create",
+      "--name=root",
+      "--symlink-mode=posix-raw",
+      "--ignore",
+      ".local/share/overlay/**",
+      "--ignore",
+      ".cache/cocalc/**",
+      "--ignore",
+      ".mutagen-dev/**",
+      "--ignore",
+      ".ssh/**",
+      "--ignore",
+      ".snapshots/**",
+      "/root",
+      `${servers[0].name}:/root`,
+    ]);
+  };
 }
 
 export async function stop({
@@ -249,13 +320,14 @@ export async function stop({
   }
 }
 
-async function podman(args: string[]) {
+async function podman(args: string[], timeout?) {
   logger.debug("podman ", args.join(" "));
   return await executeCode({
     verbose: true,
     command: "podman",
     args,
     err_on_exit: true,
+    timeout,
   });
 }