fix: generate shrinkwrap file for deterministic dependencies (#5071)

Author: edvincent

ci/build/build-release.sh

@@ -18,6 +18,8 @@ main() {
   VSCODE_SRC_PATH="lib/vscode"
   VSCODE_OUT_PATH="$RELEASE_PATH/lib/vscode"
 
+  create_shrinkwraps
+
   mkdir -p "$RELEASE_PATH"
 
   bundle_code_server
@@ -55,15 +57,6 @@ bundle_code_server() {
 EOF
   ) > "$RELEASE_PATH/package.json"
   rsync yarn.lock "$RELEASE_PATH"
-
-  # To ensure deterministic dependency versions (even when code-server is installed with NPM), we seed
-  # an npm-shrinkwrap file from our yarn lockfile and the current node-modules installed.
-  synp --source-file yarn.lock
-  npm shrinkwrap
-  # HACK@edvincent: The shrinkwrap file will contain the devDependencies, which by default
-  # are installed if present in a lockfile. To avoid every user having to specify --production
-  # to skip them, we carefully remove them from the shrinkwrap file.
-  json -f npm-shrinkwrap.json -I -e "Object.keys(this.dependencies).forEach(dependency => { if (this.dependencies[dependency].dev) { delete this.dependencies[dependency] } } )"
   mv npm-shrinkwrap.json "$RELEASE_PATH"
 
   rsync ci/build/npm-postinstall.sh "$RELEASE_PATH/postinstall.sh"
@@ -105,11 +98,44 @@ bundle_vscode() {
     "$VSCODE_SRC_PATH/package.json" > "$VSCODE_OUT_PATH/package.json"
 
   rsync "$VSCODE_SRC_PATH/remote/yarn.lock" "$VSCODE_OUT_PATH/yarn.lock"
+  mv "$VSCODE_SRC_PATH/remote/npm-shrinkwrap.json" "$VSCODE_OUT_PATH/npm-shrinkwrap.json"
 
   # Include global extension dependencies as well.
   rsync "$VSCODE_SRC_PATH/extensions/package.json" "$VSCODE_OUT_PATH/extensions/package.json"
   rsync "$VSCODE_SRC_PATH/extensions/yarn.lock" "$VSCODE_OUT_PATH/extensions/yarn.lock"
+  mv "$VSCODE_SRC_PATH/extensions/npm-shrinkwrap.json" "$VSCODE_OUT_PATH/extensions/npm-shrinkwrap.json"
   rsync "$VSCODE_SRC_PATH/extensions/postinstall.mjs" "$VSCODE_OUT_PATH/extensions/postinstall.mjs"
 }
 
+create_shrinkwraps() {
+  # yarn.lock or package-lock.json files (used to ensure deterministic versions of dependencies) are
+  # not packaged when publishing to the NPM registry.
+  # To ensure deterministic dependency versions (even when code-server is installed with NPM), we create
+  # an npm-shrinkwrap.json file from the currently installed node_modules. This ensures the versions used
+  # from development (that the yarn.lock guarantees) are also the ones installed by end-users.
+  # These will include devDependencies, but those will be ignored when installing globally (for code-server), and
+  # because we use --omit=dev when installing vscode.
+
+  # We first generate the shrinkwrap file for code-server itself - which is the current directory
+  create_shrinkwrap_keeping_yarn_lock
+
+  # Then the shrinkwrap files for the bundled VSCode
+  pushd "$VSCODE_SRC_PATH/remote/"
+  create_shrinkwrap_keeping_yarn_lock
+  popd
+
+  pushd "$VSCODE_SRC_PATH/extensions/"
+  create_shrinkwrap_keeping_yarn_lock
+  popd
+}
+
+create_shrinkwrap_keeping_yarn_lock() {
+  # HACK@edvincent: Generating a shrinkwrap alters the yarn.lock which we don't want (with NPM URLs rather than the Yarn URLs)
+  # But to generate a valid shrinkwrap, it has to exist... So we copy it to then restore it
+  cp yarn.lock yarn.lock.temp
+  npm shrinkwrap
+  cp yarn.lock.temp yarn.lock
+  rm yarn.lock.temp
+}
+
 main "$@"

ci/build/npm-postinstall.sh

@@ -110,7 +110,7 @@ main() {
     echo "Failed to download cloud agent; --link will not work"
   fi
 
-  if ! vscode_yarn; then
+  if ! vscode_install; then
     echo "You may not have the required dependencies to build the native modules."
     echo "Please see https://github.com/coder/code-server/blob/main/docs/npm.md"
     exit 1
@@ -123,17 +123,44 @@ main() {
   fi
 }
 
-vscode_yarn() {
+install_with_yarn_or_npm() {
+  # NOTE@edvincent: We want to keep using the package manager that the end-user was using to install the package.
+  # This also ensures that when *we* run `yarn` in the development process, the yarn.lock file is used.
+  case "${npm_config_user_agent-}" in
+    yarn*)
+      if [ -f "yarn.lock" ]; then
+        yarn --production --frozen-lockfile --no-default-rc
+      else
+        echo "yarn.lock file not present, not running in development mode. use npm to install code-server!"
+        exit 1
+      fi
+      ;;
+    npm*)
+      if [ -f "yarn.lock" ]; then
+        echo "yarn.lock file present, running in development mode. use yarn to install code-server!"
+        exit 1
+      else
+        npm install --omit=dev
+      fi
+      ;;
+    *)
+      echo "Could not determine which package manager is being used to install code-server"
+      exit 1
+      ;;
+  esac
+}
+
+vscode_install() {
   echo 'Installing Code dependencies...'
   cd lib/vscode
-  yarn --production --frozen-lockfile --no-default-rc
+  install_with_yarn_or_npm
 
   symlink_asar
   symlink_bin_script remote-cli code code-server
   symlink_bin_script helpers browser browser .sh
 
   cd extensions
-  yarn --production --frozen-lockfile
+  install_with_yarn_or_npm
 }
 
 main "$@"

package.json

@@ -59,13 +59,11 @@
     "eslint-import-resolver-typescript": "^2.5.0",
     "eslint-plugin-import": "^2.18.2",
     "eslint-plugin-prettier": "^4.0.0",
-    "json": "^11.0.0",
     "prettier": "^2.2.1",
     "prettier-plugin-sh": "^0.12.0",
     "shellcheck": "^1.0.0",
     "stylelint": "^13.0.0",
     "stylelint-config-recommended": "^5.0.0",
-    "synp": "^1.9.10",
     "ts-node": "^10.0.0",
     "typescript": "^4.6.2"
   },
@@ -108,8 +106,7 @@
     "semver": "^7.1.3",
     "split2": "^4.0.0",
     "ws": "^8.0.0",
-    "xdg-basedir": "^4.0.0",
-    "yarn": "^1.22.4"
+    "xdg-basedir": "^4.0.0"
   },
   "bin": {
     "code-server": "out/node/entry.js"