in theory this implements sticky routing in a way that works in all our deployment settings

Author: williamstein

src/packages/conat/core/server.ts

@@ -142,7 +142,11 @@ export interface Options {
   // if true, use https when creating an internal client.
   ssl?: boolean;
 
-  // WARNING: **superclusters are NOT fully iplemented yet.**
+  // WARNING: **superclusters are NOT implemented yet**.  Maybe they
+  // are a very bad idea and should NOT be implemented. In practice,
+  // when I hit a need for this it's better to just explicitly make
+  // a process with two clients, and use an explicit protocol to
+  // connect the clusters...
   //
   // if clusterName is set, enable clustering. Each node
   // in the cluster must have a different name. systemAccountPassword
@@ -456,8 +460,12 @@ export class ConatServer extends EventEmitter {
     this.stickyClient = this.client({
       systemAccountPassword: this.options.systemAccountPassword,
     });
-    if (!this.cluster) {
-      // not a cluster, so we can server as the sticky routing serivce
+    if (!this.cluster || this.options.localClusterSize != null) {
+      // not a cluster or a local cluster, so we view THIS server
+      // as the canonical server and it also hosts the
+      // sticky routing service.  The one case where we don't do
+      // this is for a non-local cluster, e.g., deploying on
+      // Kubernetes.
       createStickyRouter({ client: this.stickyClient });
     }
   };
@@ -1105,7 +1113,10 @@ export class ConatServer extends EventEmitter {
     if (localClusterSize < 2) {
       return;
     }
-    // spawn additional servers as separate processes to form a cluster
+    // Spawn additional servers as *separate processes* to form a cluster.
+    // This is mainly used for testing, but is also very useful on
+    // a single powerful machine where we want more routing than a single
+    // nodejs process provides. This cannot do autoscaling though.
     const port = this.options.port;
     if (!port) {
       throw Error("bug -- port must be set");

src/packages/conat/core/sticky.ts

@@ -113,6 +113,7 @@ export async function createStickyRouter({
     }
   };
   listen();
+  return sub;
 }
 
 const stickyRequest = reuseInFlight(

src/packages/hub/hub.ts

@@ -27,7 +27,10 @@ import {
   initConatPersist,
   loadConatConfiguration,
 } from "@cocalc/server/conat";
-import { initConatServer } from "@cocalc/server/conat/socketio";
+import {
+  initConatServer,
+  initStickyRouterService,
+} from "@cocalc/server/conat/socketio";
 import { init_passport } from "@cocalc/server/hub/auth";
 import { initialOnPremSetup } from "@cocalc/server/initial-onprem-setup";
 import initHandleMentions from "@cocalc/server/mentions/handle";
@@ -282,7 +285,15 @@ async function startServer(): Promise<void> {
     console.log(msg);
   }
 
+  // UNIQUE SERVICE: THESE ARE THE SERVICES THAT HAVE TO RUN *EXACTLY ONCE*.
   if (program.all || program.mentions) {
+    if (program.mode == "kucalc") {
+      // init kucalc it's critical to have one sticky router service running,
+      // and this is the hub node where we do this. Do NOT do this
+      // if there is no clustering or a local conat cluster, since then
+      // you end up with two sticky routers.
+      initStickyRouterService();
+    }
     // kucalc: for now we just have the hub-mentions servers
     // do the new project pool maintenance, since there is only
     // one hub-stats.

src/packages/server/conat/socketio/index.ts

@@ -1 +1,10 @@
 export { init as initConatServer } from "./server";
+import { loadConatConfiguration } from "../configuration";
+import { conat } from "@cocalc/backend/conat";
+import { createStickyRouter } from "@cocalc/conat/core/sticky";
+
+export async function initStickyRouterService() {
+  await loadConatConfiguration();
+  const client = conat();
+  createStickyRouter({ client });
+}