address obvious symlink issue with fs sandbox; also make tests hopefully work on github

Author: williamstein

src/package.json

@@ -18,7 +18,7 @@
     "version-check": "pip3 install typing_extensions mypy  || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py",
     "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test",
     "test": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test",
-    "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter --retries=1",
+    "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --test-github-ci --exclude=jupyter --retries=1",
     "depcheck": "cd packages && pnpm run -r --parallel depcheck",
     "prettier-all": "cd packages/",
     "local-ci": "./scripts/ci.sh",

src/packages/file-server/btrfs/filesystem.ts

@@ -69,6 +69,7 @@ export class Filesystem {
       args: ["quota", "enable", "--simple", this.opts.mount],
     });
     await this.initBup();
+    await this.sync();
   };
 
   sync = async () => {

src/packages/file-server/btrfs/subvolume-bup.ts

@@ -46,7 +46,7 @@ export class SubvolumeBup {
         `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`,
       );
       await this.subvolume.snapshots.create(BUP_SNAPSHOT);
-      const target = this.subvolume.fs.safeAbsPath(
+      const target = await this.subvolume.fs.safeAbsPath(
         this.subvolume.snapshots.path(BUP_SNAPSHOT),
       );
 
@@ -133,9 +133,9 @@ export class SubvolumeBup {
       return v;
     }
 
-    path = this.subvolume.fs
-      .safeAbsPath(path)
-      .slice(this.subvolume.path.length);
+    path = (await this.subvolume.fs.safeAbsPath(path)).slice(
+      this.subvolume.path.length,
+    );
     const { stdout } = await sudo({
       command: "bup",
       args: [

src/packages/file-server/btrfs/subvolume.ts

@@ -86,8 +86,8 @@ export class Subvolume {
     target: string;
     timeout?: number;
   }): Promise<{ stdout: string; stderr: string; exit_code: number }> => {
-    let srcPath = this.fs.safeAbsPath(src);
-    let targetPath = this.fs.safeAbsPath(target);
+    let srcPath = await this.fs.safeAbsPath(src);
+    let targetPath = await this.fs.safeAbsPath(target);
     if (src.endsWith("/")) {
       srcPath += "/";
     }

src/packages/file-server/btrfs/test/filesystem-stress.test.ts

@@ -48,7 +48,6 @@ describe("stress operations with subvolumes", () => {
   });
 
   it("clone the first group in serial", async () => {
-    await fs.sync(); // needed on github actions
     const t = Date.now();
     for (let i = 0; i < count1; i++) {
       await fs.subvolumes.clone(`${i}`, `clone-of-${i}`);
@@ -59,7 +58,6 @@ describe("stress operations with subvolumes", () => {
   });
 
   it("clone the second group in parallel", async () => {
-    await fs.sync(); // needed on github actions
     const t = Date.now();
     const v: any[] = [];
     for (let i = 0; i < count2; i++) {
@@ -94,7 +92,6 @@ describe("stress operations with subvolumes", () => {
   });
 
   it("everything should be gone except the clones", async () => {
-    await fs.sync();
     const v = await fs.subvolumes.list();
     expect(v.length).toBe(count1 + count2);
   });

src/packages/file-server/btrfs/test/subvolume.test.ts

@@ -143,7 +143,7 @@ describe("the filesystem operations", () => {
     await vol.fs.writeFile("w.txt", "hi");
     const ac = new AbortController();
     const { signal } = ac;
-    const watcher = vol.fs.watch("w.txt", { signal });
+    const watcher = await vol.fs.watch("w.txt", { signal });
     vol.fs.appendFile("w.txt", " there");
     // @ts-ignore
     const { value, done } = await watcher.next();

src/packages/file-server/conat/test/local-path.test.ts

@@ -1,25 +1,28 @@
 import { localPathFileserver } from "../local-path";
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdtemp, readFile, rm, symlink } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "path";
 import { fsClient } from "@cocalc/conat/files/fs";
 import { randomId } from "@cocalc/conat/names";
 import { before, after, client } from "@cocalc/backend/conat/test/setup";
+import { uuid } from "@cocalc/util/misc";
 
 let tempDir;
+let tempDir2;
 beforeAll(async () => {
   await before();
   tempDir = await mkdtemp(join(tmpdir(), "cocalc-local-path"));
+  tempDir2 = await mkdtemp(join(tmpdir(), "cocalc-local-path-2"));
 });
 
-describe("use the simple fileserver", () => {
+describe("use all the standard api functions of fs", () => {
   const service = `fs-${randomId()}`;
   let server;
   it("creates the simple fileserver service", async () => {
     server = await localPathFileserver({ client, service, path: tempDir });
   });
 
-  const project_id = "6b851643-360e-435e-b87e-f9a6ab64a8b1";
+  const project_id = uuid();
   let fs;
   it("create a client", () => {
     fs = fsClient({ subject: `${service}.project-${project_id}` });
@@ -82,6 +85,27 @@ describe("use the simple fileserver", () => {
     expect(s.isFile()).toBe(false);
   });
 
+  it("readFile works", async () => {
+    await fs.writeFile("a", Buffer.from([1, 2, 3]));
+    const s = await fs.readFile("a");
+    expect(s).toEqual(Buffer.from([1, 2, 3]));
+
+    await fs.writeFile("b.txt", "conat");
+    const t = await fs.readFile("b.txt", "utf8");
+    expect(t).toEqual("conat");
+  });
+
+  it("readdir works", async () => {
+    await fs.mkdir("dirtest");
+    for (let i = 0; i < 5; i++) {
+      await fs.writeFile(`dirtest/${i}`, `${i}`);
+    }
+    const fire = "🔥.txt";
+    await fs.writeFile(join("dirtest", fire), "this is ️‍🔥!");
+    const v = await fs.readdir("dirtest");
+    expect(v).toEqual(["0", "1", "2", "3", "4", fire]);
+  });
+
   it("creating a symlink works (and using lstat)", async () => {
     await fs.writeFile("source1", "the source");
     await fs.symlink("source1", "target1");
@@ -99,15 +123,71 @@ describe("use the simple fileserver", () => {
     const stats0 = await fs.stat("source1");
     expect(stats0.isSymbolicLink()).toBe(false);
   });
-  
-  
 
   it("closes the service", () => {
     server.close();
   });
 });
 
+describe("security: dangerous symlinks can't be followed", () => {
+  const service = `fs-${randomId()}`;
+  let server;
+  it("creates the simple fileserver service", async () => {
+    server = await localPathFileserver({ client, service, path: tempDir2 });
+  });
+
+  const project_id = uuid();
+  const project_id2 = uuid();
+  let fs, fs2;
+  it("create two clients", () => {
+    fs = fsClient({ subject: `${service}.project-${project_id}` });
+    fs2 = fsClient({ subject: `${service}.project-${project_id2}` });
+  });
+
+  it("create a secret in one", async () => {
+    await fs.writeFile("password", "s3cr3t");
+    await fs2.writeFile("a", "init");
+  });
+
+  // This is setup bypassing security and is part of our threat model, due to users
+  // having full access internally to their sandbox fs.
+  it("directly create a file that is a symlink outside of the sandbox -- this should work", async () => {
+    await symlink(
+      join(tempDir2, project_id, "password"),
+      join(tempDir2, project_id2, "link"),
+    );
+    const s = await readFile(join(tempDir2, project_id2, "link"), "utf8");
+    expect(s).toBe("s3cr3t");
+  });
+
+  it("fails to read the symlink content via the api", async () => {
+    await expect(async () => {
+      await fs2.readFile("link", "utf8");
+    }).rejects.toThrow("outside of sandbox");
+  });
+
+  it("directly create a relative symlink ", async () => {
+    await symlink(
+      join("..", project_id, "password"),
+      join(tempDir2, project_id2, "link2"),
+    );
+    const s = await readFile(join(tempDir2, project_id2, "link2"), "utf8");
+    expect(s).toBe("s3cr3t");
+  });
+
+  it("fails to read the relative symlink content via the api", async () => {
+    await expect(async () => {
+      await fs2.readFile("link2", "utf8");
+    }).rejects.toThrow("outside of sandbox");
+  });
+
+  it("closes the server", () => {
+    server.close();
+  });
+});
+
 afterAll(async () => {
   await after();
   await rm(tempDir, { force: true, recursive: true });
+  // await rm(tempDir2, { force: true, recursive: true });
 });