Merge remote-tracking branch 'origin/master' into cocalc-api-20250927

Author: haraldschilly

src/AGENTS.md

@@ -292,17 +292,10 @@ The `python/cocalc-api/` directory contains a Python client library for the CoCa
 - ✅ Complete project lifecycle management: create → start → test → stop → delete
 - ✅ All 14 tests passing with proper resource cleanup
 
-# Important Instruction Reminders
-
-- Do what has been asked; nothing more, nothing less
-- NEVER create files unless they're absolutely necessary for achieving your goal
-- ALWAYS prefer editing an existing file to creating a new one
-- REFUSE to modify files when the git repository is on the `master` or `main` branch
-- NEVER proactively create documentation files (`*.md`) or README files. Only create documentation files if explicitly requested by the User
-
 # important-instruction-reminders
-Do what has been asked; nothing more, nothing less.
-NEVER create files unless they're absolutely necessary for achieving your goal.
-ALWAYS prefer editing an existing file to creating a new one.
-NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.
-ALWAYS ask questions if something is unclear. Only proceed to the implementation step if you have no questions left.
+- Do what has been asked; nothing more, nothing less.
+- NEVER create files unless they're absolutely necessary for achieving your goal.
+- ALWAYS prefer editing an existing file to creating a new one.
+- NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.
+- ALWAYS ask questions if something is unclear. Only proceed to the implementation step if you have no questions left.
+- When modifying a file with a copyright banner at the top, make sure to fix/add the current year to indicate the copyright year.

src/packages/backend/conat/test/cluster/cluster-sticky-state.test.ts

@@ -3,6 +3,7 @@ import {
   after,
   defaultCluster as servers,
   waitForConsistentState,
+  wait,
   addNodeToDefaultCluster,
 } from "@cocalc/backend/conat/test/setup";
 import { STICKY_QUEUE_GROUP } from "@cocalc/conat/core/client";
@@ -45,6 +46,9 @@ describe("ensure sticky state sync and use is working properly", () => {
       );
       // publishing causes a choice to be made and saved on servers[0]
       await clients[0].publish(`subject.${i}.foo`, "hello");
+      expect(servers[0].sticky[`subject.${i}.*`]).not.toBe(undefined);
+      // but no choice on servers[1]
+      expect(servers[1].sticky[`subject.${i}.*`]).toBe(undefined);
     }
   });
 
@@ -61,12 +65,51 @@ describe("ensure sticky state sync and use is working properly", () => {
     chosen = await Promise.race([p0(), p1()]);
   });
 
+  it(`sticky on servers[0] should have ${count} entries starting in "subject".`, async () => {
+    const v = Object.keys(servers[0].sticky).filter((s) =>
+      s.startsWith("subject."),
+    );
+    expect(v.length).toBe(count);
+  });
+
+  it(`sticky on servers[1] should have no entries starting in "subject".`, async () => {
+    const v = Object.keys(servers[1].sticky).filter((s) =>
+      s.startsWith("subject."),
+    );
+    expect(v.length).toBe(0);
+  });
+
+  it(`servers[1]'s link to servers[0] should *eventually* have ${count} entries starting in "subject."`, async () => {
+    // @ts-ignore
+    const link = servers[1].clusterLinksByAddress[servers[0].address()];
+    let v;
+    await wait({
+      until: () => {
+        v = Object.keys(link.sticky).filter((s) => s.startsWith("subject."));
+        return v.length == count;
+      },
+    });
+    expect(v.length).toBe(count);
+  });
+
   it("send message from clients[1] to each subject", async () => {
     for (let i = 0; i < count; i++) {
       await clients[1].publish(`subject.${i}.foo`);
     }
   });
 
+  // Sometimes this fails under very heavy load.
+  // It's not a good test, because it probably hits some timeouts sometimes, and
+  // it is testing internal structure/optimizations, not behavior.
+  // Note also that minimizing sticky state computation is just an optimization so even if this test were failing
+  // due to a bug, it might just mean things are slightly slower.
+  //   it(`sticky on servers[1] should STILL have no entries starting in "subject", since no choices had to be made`, async () => {
+  //     const v = Object.keys(servers[1].sticky).filter((s) =>
+  //       s.startsWith("subject."),
+  //     );
+  //     expect(v.length).toBe(0);
+  //   });
+
   async function deliveryTest() {
     const sub = chosen == 0 ? subs0[0] : subs1[0];
 
@@ -111,6 +154,17 @@ describe("ensure sticky state sync and use is working properly", () => {
     await waitForConsistentState(servers);
   });
 
+  it("double check the links have the sticky state", () => {
+    for (const server of servers.slice(1)) {
+      // @ts-ignore
+      const link = server.clusterLinksByAddress[servers[0].address()];
+      const v = Object.keys(link.sticky).filter((s) =>
+        s.startsWith("subject."),
+      );
+      expect(v.length).toBe(count);
+    }
+  });
+
   it(
     "in bigger, cluster, publish from every node to subject.0.foo",
     deliveryTest,
@@ -126,6 +180,14 @@ describe("ensure sticky state sync and use is working properly", () => {
     }
     sub.close();
   });
+
+  it("unjoining servers[0] from servers[1] should transfer the sticky state to servers[1]", async () => {
+    await servers[1].unjoin({ address: servers[0].address() });
+    const v = Object.keys(servers[1].sticky).filter((s) =>
+      s.startsWith("subject."),
+    );
+    expect(v.length).toBe(count);
+  });
 });
 
 afterAll(after);

src/packages/backend/conat/test/setup.ts

@@ -206,6 +206,7 @@ export async function waitForConsistentState(
         // now look at everybody else's view of servers[i].
         // @ts-ignore
         const a = servers[i].interest.serialize().patterns;
+        const b = servers[i].sticky;
         const hashServer = servers[i].hash();
         for (let j = 0; j < servers.length; j++) {
           if (i != j) {
@@ -219,8 +220,9 @@ export async function waitForConsistentState(
             }
             const hashLink = link.hash();
             const x = link.interest.serialize().patterns;
+            const y = link.sticky;
             const showInfo = () => {
-              for (const type of ["interest"]) {
+              for (const type of ["interest", "sticky"]) {
                 console.log(
                   `server stream ${type}: `,
                   hashServer[type],
@@ -252,6 +254,8 @@ export async function waitForConsistentState(
                 j,
                 serverInterest: a,
                 linkInterest: x,
+                serverSticky: b,
+                linkSticky: y,
               });
             };
             if (!isEqual(hashServer, hashLink)) {

src/packages/backend/conat/test/sync/dkv-merge.test.ts

@@ -9,7 +9,7 @@ pnpm test ./dkv-merge.test.ts
 
 import { dkv as createDkv } from "@cocalc/backend/conat/sync";
 import { once } from "@cocalc/util/async-utils";
-import { diff_match_patch } from "@cocalc/util/dmp";
+import { DiffMatchPatch } from "@cocalc/util/dmp";
 import { before, after } from "@cocalc/backend/conat/test/setup";
 
 beforeAll(before);
@@ -118,7 +118,7 @@ describe("test a trivial merge conflict resolution function", () => {
 });
 
 describe("test a 3-way merge of strings conflict resolution function", () => {
-  const dmp = new diff_match_patch();
+  const dmp = new DiffMatchPatch();
   const threeWayMerge = (opts: {
     prev: string;
     local: string;

src/packages/backend/conat/test/sync/open-files.test.ts

@@ -8,7 +8,7 @@ to open so they can fulfill their backend responsibilities:
 
 DEVELOPMENT:
 
-pnpm test `pwd`/open-files.test.ts
+pnpm test ./open-files.test.ts
 
 */
 

src/packages/backend/data.ts

@@ -171,14 +171,28 @@ export function sslConfigToPsqlEnv(config: SSLConfig): PsqlSSLEnvConfig {
 
 export const root: string = process.env.COCALC_ROOT ?? determineRootFromPath();
 export const data: string = process.env.DATA ?? join(root, "data");
+
+// Database Config
 export const pguser: string = process.env.PGUSER ?? "smc";
 export const pgdata: string = process.env.PGDATA ?? join(data, "postgres");
 export const pghost: string = process.env.PGHOST ?? join(pgdata, "socket");
 export const pgssl = sslConfigFromCoCalcEnv();
 export const pgdatabase: string =
   process.env.SMC_DB ?? process.env.PGDATABASE ?? "smc";
+
+// Database healthchecks
+export const pgConcurrentWarn: number = parseInt(
+  process.env.COCALC_HEALTH_PG_CONCURRENT_WARN ?? "300",
+);
+
+// Hub server configuration
+export const hubHostname: string =
+  process.env.COCALC_HUB_HOSTNAME ?? "127.0.0.1";
+export const agentPort: number = parseInt(process.env.COCALC_AGENT_PORT ?? "0");
+
 export const projects: string =
   process.env.PROJECTS ?? join(data, "projects", "[project_id]");
+
 export const secrets: string = process.env.SECRETS ?? join(data, "secrets");
 
 // Where the sqlite database files used for sync are stored.