Merge remote-tracking branch 'origin/master' into sso-exclusive-all

Author: haraldschilly

src/compute/compute/dev/4-startup-script.sh

@@ -17,19 +17,21 @@
 
 set -v
 
-export api_server=`cat conf/api_server`
-
+. env.sh
 
 function setState {
-  id=`cat conf/compute_server_id`
+  id=$COMPUTE_SERVER_ID
   name=$1
   state=${2:-'ready'}
   extra=${3:-''}
   timeout=${4:-0}
   progress=${5:-100}
+  project_id=$PROJECT_ID
 
   echo "$name is $state"
-  curl -sk -u `cat conf/api_key`:  -H 'Content-Type: application/json' -d "{\"id\":$id,\"name\":\"$name\",\"state\":\"$state\",\"extra\":\"$extra\",\"timeout\":$timeout,\"progress\":$progress}" $api_server/api/v2/compute/set-detailed-state
+  PAYLOAD="{\"id\":$id,\"name\":\"$name\",\"state\":\"$state\",\"extra\":\"$extra\",\"timeout\":$timeout,\"progress\":$progress,\"project_id\":\"$project_id\"}"
+  echo $PAYLOAD
+  curl -sk -u $API_KEY:  -H 'Content-Type: application/json' -d $PAYLOAD $API_SERVER/api/v2/compute/set-detailed-state
 }
 
 
@@ -53,6 +55,8 @@ setState vm start '' 60 60
 sleep 0.1
 
 while true; do
+  setState compute ready '' 35 100
+  setState filesystem-sync ready '' 35 100
   setState vm ready '' 35 100
   sleep 30
 done

src/compute/compute/dev/README.md

@@ -2,11 +2,17 @@ The scripts here are helpful for developing the compute\-server manager, which i
 
 1. Create the directory /tmp/user and make sure you can read it. Maybe even mount it from the target project.
 
-2. Make the conf/ directory here, with the same files as on /cocalc/conf in an actual compute\-server, except:
-   - replace api_server by something like `http://localhost:5000/6659c2e3-ff5e-4bb4-9a43-8830aa951282/port/5000`, where the port is what you're using for your dev server and the project id is of your dev server. The point is that we're going to connect directly without going through some external server.
-   - api_key: the one from an actual server will get deleted when you turn that server off, so make a different project level api key.
+2. The conf/ directory here has the same files as on /cocalc/conf in an actual compute\-server, except:
+     - replace api_server by something like `http://127.0.0.1:5000/6659c2e3-ff5e-4bb4-9a43-8830aa951282/port/5000`, where the port is what you're using for your dev server and the project id is of your dev server. The point is that we're going to connect directly without going through some external server.
+     - api_key: the one from an actual server will get deleted when you turn that server off, so make a different project level api key.
 
-This is obviously very confusing, and when developing this it was 10x worse... Maybe you'll be confused for 2 hours instead of 2 days.
+   Type `tar xvf conf.tar` to get a template for the conf directory.  
+   You will need to change the contents of all the files you get, as 
+   mentioned above!  Also, regarding the api_server, be especially careful
+   about ipv4 versus ipv6, e.g., use 127.0.0.1 instead of localhost to
+   nail down the protocol.
+
+This is potentially confusing, and when developing this it was 10x worse... Maybe you'll be confused for 2 hours instead of 2 days.
 
 3. Run each of the following four shell scripts in different terminals, in order.
 
@@ -17,17 +23,27 @@ This is obviously very confusing, and when developing this it was 10x worse... M
 4-startup-script.sh
 ```
 
-However, a bunch of things are likely to go wrong. The scripts `1-websocketfs.sh` and `2-syncfs.sh` will definitely fail if support for FUSE isn't enabled for normal users where you are working! Test bindfs locally. You probably  have to add `user_allow_other` to `/etc/fuse.conf`. For `2-syncfs.sh`, you must also install unionfs-fuse via `sudo apt install unionfs-fuse` . Also, you need to do the following so that testing of tmp being a "fast local data directory that isn't sync"'d can be done:
+However, a bunch of things are likely to go wrong. 
 
-```
-~/cocalc/src/compute/compute/dev$ sudo mkdir /data/tmp
-~/cocalc/src/compute/compute/dev$ sudo chown `whoami`:`whoami` /data/tmp
+**Problem:** Regarding the id of the compute server in the file [conf/compute\_server\_id](./conf/compute_server_id), create a self\-hosted compute server in the project on your dev server, then find the record in the postgresql database by querying the `compute_servers` table, and copy the id field from that.  Note that the displayed id in the UI starts from 1 for each project, but `compute_server_id` must be the id in the database.
+
+**Problem:** Get the [conf/api_key](./conf/api_key) by clicking start on the self\-hosted compute server, inspect the URL, and copy it from there.  If you stop the server explicitly, then the api key is deleted from the project, so you need to make it again.
+
+**Problem:** The scripts `1-websocketfs.sh` and `2-syncfs.sh` will definitely fail if support for FUSE isn't enabled for normal users where you are working! Test bindfs locally. 
+
+**Problem:** For `2-syncfs.sh`, you must also install unionfs\-fuse via `sudo apt install unionfs-fuse,`  since the cocalc package @cocalc/sync\-fs assumes unionfs\-fuse is installed.   
+
+**Problem:** You need to do the following so that you can fully test the scratch functionality \(see [conf/exclude_from_sync](./conf/exclude_from_sync)\):
+
+```sh
+sudo mkdir -p /data/scratch && sudo chown -R `whoami`:`whoami` /data
 ```
 
-Once you get the 4 scripts above to run, the net result is basically the same as using a compute server, but you can run it all locally, and debugging is massively easier. Without something like this, development is impossible, and even figuring out what configuration goes where could cost me days of confusion (even though I wrote it all!). It's complicated.
+Once you get the 4 scripts above to run, the net result is basically the same as using a compute server, but you can run it all locally, and development and debugging is ~~massively easier~~ possible! Without something like this, development is impossible, and even figuring out what configuration goes where could cost me days of confusion \(even though I wrote it all!\). It's complicated.
 
 For debugging set the DEBUG env variable to different things according to the debug npm module. E.g.,
 
 ```sh
-DEBUG=* 2-syncfs.sh
+DEBUG_CONSOLE=yes DEBUG=* ./2-syncfs.sh
 ```
+

src/compute/compute/dev/conf.tar

@@ -47,7 +47,8 @@ async function main() {
     exports.fs = await mountProject({
       project_id: process.env.PROJECT_ID,
       path: PROJECT_HOME,
-      options: { mountOptions: { allowOther: true, nonEmpty: true } },
+      // NOTE: allowOther is disabled by default on Ubuntu and we do not need it. 
+      options: { mountOptions: { allowOther: false, nonEmpty: true } },
       unionfs,
       readTrackingFile: process.env.READ_TRACKING_FILE,
       exclude,
@@ -58,8 +59,9 @@ async function main() {
     });
     unmount = exports.fs.unmount;
   } catch (err) {
-    console.log("something went wrong ", err);
+    console.trace("something went wrong ", err);
     exitHandler();
+    return;
   }
 
   const info = () => {

src/compute/compute/dev/start-filesystem.js

@@ -11,7 +11,6 @@ import { mount } from "websocketfs";
 import getLogger from "@cocalc/backend/logger";
 import { project } from "@cocalc/api-client";
 import { serialize } from "cookie";
-import { join } from "path";
 import { API_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names";
 import syncFS from "@cocalc/sync-fs";
 import {
@@ -100,7 +99,7 @@ export async function mountProject({
     // Ping to start project so it's possible to mount.
     await pingProjectUntilSuccess(project_id);
 
-    const remote = join(getProjectWebsocketUrl(project_id), "websocketfs");
+    const remote = getProjectWebsocketUrl(project_id) + "/websocketfs";
     log("connecting to ", remote);
     const headers = { Cookie: serialize(API_COOKIE_NAME, apiKey) };
     // SECURITY: DO NOT log headers and connectOptions, obviously!
@@ -139,7 +138,7 @@ export async function mountProject({
         progress: 30,
       });
 
-      ({ unmount } = await mount({
+      const websocketfsMountOptions = {
         remote,
         path: homeMountPoint,
         ...options,
@@ -149,9 +148,9 @@ export async function mountProject({
           ...options.connectOptions,
         },
         mountOptions: {
-          allowOther: true,
-          nonEmpty: true,
           ...options.mountOptions,
+          allowOther: true, // this is critical to allow for fast bind mounts of scratch etc. as root.
+          nonEmpty: true,
         },
         cacheTimeout,
         hidePath: "/.unionfs",
@@ -163,7 +162,26 @@ export async function mountProject({
         readTrackingExclude: exclude,
         // metadata file
         metadataFile,
-      }));
+      };
+
+      log("websocketfs -- mount options", websocketfsMountOptions);
+
+      try {
+        ({ unmount } = await mount(websocketfsMountOptions));
+      } catch (err) {
+        log("failed trying to mount -- ", err);
+        log(
+          "try again without allowOther, since some versions of FUSE do not support this option",
+        );
+        websocketfsMountOptions.mountOptions.allowOther = false;
+        ({ unmount } = await mount(websocketfsMountOptions));
+
+        // This worked so the problem is allow_other.
+        throw Error(
+          "fusermount: option allow_other only allowed if 'user_allow_other' is set in /etc/fuse.conf\n\n\nFix this:\n\n    sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf\n\n\n",
+        );
+      }
+
       pingInterval = setInterval(async () => {
         try {
           await project.ping({ project_id });

src/compute/compute/lib/filesystem.ts

@@ -338,9 +338,12 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
         return await update_account_and_passport(@, opts)
 
     ###
-    Account creation, deletion, existence
+    Creating an account using SSO only.
+    This needs to be rewritten in @cocalc/server like
+    all the other account creation.  This is horrible
+    because 
     ###
-    create_account: (opts={}) =>
+    create_sso_account: (opts={}) =>
         opts = defaults opts,
             first_name        : undefined
             last_name         : undefined
@@ -357,7 +360,7 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
             usage_intent      : undefined
             cb                : required       # cb(err, account_id)
 
-        dbg = @_dbg("create_account(#{opts.first_name}, #{opts.last_name}, #{opts.lti_id}, #{opts.email_address}, #{opts.passport_strategy}, #{opts.passport_id}), #{opts.usage_intent}")
+        dbg = @_dbg("create_sso_account(#{opts.first_name}, #{opts.last_name}, #{opts.lti_id}, #{opts.email_address}, #{opts.passport_strategy}, #{opts.passport_id}), #{opts.usage_intent}")
         dbg()
 
         for name in ['first_name', 'last_name']
@@ -974,27 +977,6 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
                 @record_file_use(project_id:opts.project_id, path:opts.path, action:opts.action, account_id:opts.account_id, cb:cb)
         ], (err)->opts.cb?(err))
 
-    ###
-    Rememberme cookie functionality
-    ###
-    # Save remember me info in the database
-    save_remember_me: (opts) =>
-        opts = defaults opts,
-            account_id : required
-            hash       : required
-            value      : required
-            ttl        : required
-            cb         : required
-        if not @_validate_opts(opts) then return
-        @_query
-            query : 'INSERT INTO remember_me'
-            values :
-                'hash       :: TEXT      ' : opts.hash.slice(0,127)
-                'value      :: JSONB     ' : opts.value
-                'expire     :: TIMESTAMP ' : expire_time(opts.ttl)
-                'account_id :: UUID      ' : opts.account_id
-            conflict : 'hash'
-            cb       : opts.cb
 
     # Invalidate all outstanding remember me cookies for the given account by
     # deleting them from the remember_me key:value store.

src/packages/database/postgres-server-queries.coffee

@@ -191,15 +191,15 @@ export interface PostgreSQL extends EventEmitter {
   set_server_setting(opts: { name: string; value: string; cb: CB }): void;
   server_settings_synctable(): any; // returns a table
 
-  create_account(opts: {
+  create_sso_account(opts: {
     first_name?: string; // invalid name will throw Error
     last_name?: string; // invalid name will throw Error
     created_by?: string;
     email_address?: string;
     password_hash?: string;
-    passport_strategy?: any;
-    passport_id?: string;
-    passport_profile?: any;
+    passport_strategy: any;
+    passport_id: string;
+    passport_profile: any;
     usage_intent?: string;
     cb: CB;
   }): void;
@@ -224,14 +224,6 @@ export interface PostgreSQL extends EventEmitter {
 
   get_remember_me(opts: { hash: string; cb: CB });
 
-  save_remember_me(opts: {
-    account_id: string;
-    hash: string;
-    value: string;
-    ttl: number;
-    cb: CB;
-  });
-
   passport_exists(opts: PassportExistsOpts): Promise<string | undefined>;
 
   create_passport(opts: CreatePassportOpts): Promise<string>;

src/packages/database/postgres/types.ts

@@ -31,6 +31,7 @@ export interface PassportLoginOpts {
   update_on_login: boolean; // passed down from StrategyConf, default false
   cookie_ttl_s?: number; // how long the remember_me cookied lasts (default is a month or so)
   host: string;
+  site_url: string;
   cb?: (err) => void;
 }
 

src/packages/database/settings/auth-sso-types.ts

@@ -175,7 +175,7 @@ export const DefaultAccountDropDownLinks: React.FC<LinksProps> = ({
               className={"cocalc-account-button"}
               href=""
             >
-              xSign out...
+              Sign out...
             </a>
           </Popconfirm>
         </li>