[JB] Properly overide jb config and cache folder locations (#19394)

* Revert "Revert jb stable images to 2023.3.2 (#19389)"

This reverts commit 49269b9.

* Update platform properties file

* Fix

* Update Platform Version of JetBrains Gateway Plugin (Stable) to 233.14015-EAP-CANDIDATE-SNAPSHOT

* [JetBrains] Update IDE images to new build version

* 💄

* Fix plugin installation and config sync

* Check before updating platform properties file

* Fix

* 💄

* Update jb integration test

* Fix test

* Add unit test

---------

Co-authored-by: Robo Quat <roboquat@gitpod.io>

Author: jeanp413

WORKSPACE.yaml

@@ -11,14 +11,14 @@ defaultArgs:
   codeVersion: 1.86.0
   codeQuality: stable
   noVerifyJBPlugin: false
-  intellijDownloadUrl: "https://download.jetbrains.com/idea/ideaIU-2023.3.2.tar.gz"
-  golandDownloadUrl: "https://download.jetbrains.com/go/goland-2023.3.2.tar.gz"
-  pycharmDownloadUrl: "https://download.jetbrains.com/python/pycharm-professional-2023.3.2.tar.gz"
-  phpstormDownloadUrl: "https://download.jetbrains.com/webide/PhpStorm-2023.3.2.tar.gz"
-  rubymineDownloadUrl: "https://download.jetbrains.com/ruby/RubyMine-2023.3.2.tar.gz"
-  webstormDownloadUrl: "https://download.jetbrains.com/webstorm/WebStorm-2023.3.2.tar.gz"
-  riderDownloadUrl: "https://download.jetbrains.com/rider/JetBrains.Rider-2023.3.2.tar.gz"
-  clionDownloadUrl: "https://download.jetbrains.com/cpp/CLion-2023.3.2.tar.gz"
+  intellijDownloadUrl: "https://download.jetbrains.com/idea/ideaIU-2023.3.3.tar.gz"
+  golandDownloadUrl: "https://download.jetbrains.com/go/goland-2023.3.3.tar.gz"
+  pycharmDownloadUrl: "https://download.jetbrains.com/python/pycharm-professional-2023.3.3.tar.gz"
+  phpstormDownloadUrl: "https://download.jetbrains.com/webide/PhpStorm-2023.3.3.tar.gz"
+  rubymineDownloadUrl: "https://download.jetbrains.com/ruby/RubyMine-2023.3.3.tar.gz"
+  webstormDownloadUrl: "https://download.jetbrains.com/webstorm/WebStorm-2023.3.3.tar.gz"
+  riderDownloadUrl: "https://download.jetbrains.com/rider/JetBrains.Rider-2023.3.3.tar.gz"
+  clionDownloadUrl: "https://download.jetbrains.com/cpp/CLion-2023.3.3.tar.gz"
   jbBackendVersion: "latest"
   dockerVersion: "20.10.24"
   dockerComposeVersion: "2.23.3-gitpod.1"

components/ide/jetbrains/backend-plugin/gradle-stable.properties

@@ -1,10 +1,10 @@
 # this file is auto generated by components/ide/jetbrains/image/gha-update-image/index.js
 # See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
 # for insight into build numbers and IntelliJ Platform versions.
-pluginSinceBuild=233.13135
+pluginSinceBuild=233.14015
 pluginUntilBuild=233.*
 # Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl
 # See https://jb.gg/intellij-platform-builds-list for available build versions.
 pluginVerifierIdeVersions=2023.3
 # Version from "com.jetbrains.intellij.idea" which can be found at https://www.jetbrains.com/intellij-repository/snapshots
-platformVersion=233.13135-EAP-CANDIDATE-SNAPSHOT
+platformVersion=233.14015-EAP-CANDIDATE-SNAPSHOT

components/ide/jetbrains/gateway-plugin/gradle-stable.properties

@@ -1,9 +1,9 @@
 # See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
 # for insight into build numbers and IntelliJ Platform versions.
-pluginSinceBuild=233.13135
+pluginSinceBuild=233.14015
 pluginUntilBuild=233.*
 # Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl
 # See https://jb.gg/intellij-platform-builds-list for available build versions.
 pluginVerifierIdeVersions=2023.3
 # Version from "com.jetbrains.gateway" which can be found at https://www.jetbrains.com/intellij-repository/snapshots
-platformVersion=233.13135-EAP-CANDIDATE-SNAPSHOT
+platformVersion=233.14015-EAP-CANDIDATE-SNAPSHOT

components/ide/jetbrains/launcher/main.go

@@ -65,13 +65,13 @@ type LaunchContext struct {
 	backendVersion *version.Version
 	wsInfo         *supervisor.WorkspaceInfoResponse
 
-	vmOptionsFile     string
-	projectDir        string
-	configDir         string
-	systemDir         string
-	projectConfigDir  string
-	projectContextDir string
-	riderSolutionFile string
+	vmOptionsFile          string
+	platformPropertiesFile string
+	projectDir             string
+	configDir              string
+	systemDir              string
+	projectContextDir      string
+	riderSolutionFile      string
 
 	env []string
 }
@@ -462,34 +462,36 @@ func launch(launchCtx *LaunchContext) {
 	}
 
 	launchCtx.projectDir = projectDir
-	launchCtx.configDir = fmt.Sprintf("/workspace/.config/JetBrains%s", launchCtx.qualifier)
-	launchCtx.systemDir = fmt.Sprintf("/workspace/.cache/JetBrains%s", launchCtx.qualifier)
+	launchCtx.configDir = fmt.Sprintf("/workspace/.config/JetBrains%s/RemoteDev-%s", launchCtx.qualifier, launchCtx.info.ProductCode)
+	launchCtx.systemDir = fmt.Sprintf("/workspace/.cache/JetBrains%s/RemoteDev-%s", launchCtx.qualifier, launchCtx.info.ProductCode)
 	launchCtx.riderSolutionFile = riderSolutionFile
 	launchCtx.projectContextDir = resolveProjectContextDir(launchCtx)
-	launchCtx.projectConfigDir = fmt.Sprintf("%s/RemoteDev-%s/%s", launchCtx.configDir, launchCtx.info.ProductCode, strings.ReplaceAll(launchCtx.projectContextDir, "/", "_"))
 
-	alreadySync, err := syncInitialContent(launchCtx, Options)
+	launchCtx.platformPropertiesFile = launchCtx.backendDir + "/bin/idea.properties"
+	_, err = configurePlatformProperties(launchCtx.platformPropertiesFile, launchCtx.configDir, launchCtx.systemDir)
+	if err != nil {
+		log.WithError(err).Error("failed to update platform properties file")
+	}
+
+	_, err = syncInitialContent(launchCtx, Options)
 	if err != nil {
 		log.WithError(err).Error("failed to sync initial options")
 	}
 
-	launchCtx.env = resolveLaunchContextEnv(launchCtx.configDir, launchCtx.systemDir, !alreadySync)
+	launchCtx.env = resolveLaunchContextEnv()
 
 	_, err = syncInitialContent(launchCtx, Plugins)
 	if err != nil {
 		log.WithError(err).Error("failed to sync initial plugins")
 	}
 
 	// install project plugins
-	version_2022_1, _ := version.NewVersion("2022.1")
-	if version_2022_1.LessThanOrEqual(launchCtx.backendVersion) {
-		err = installPlugins(gitpodConfig, launchCtx)
-		installPluginsCost := time.Now().Local().Sub(launchCtx.startTime).Milliseconds()
-		if err != nil {
-			log.WithError(err).WithField("cost", installPluginsCost).Error("installing repo plugins: done")
-		} else {
-			log.WithField("cost", installPluginsCost).Info("installing repo plugins: done")
-		}
+	err = installPlugins(gitpodConfig, launchCtx)
+	installPluginsCost := time.Now().Local().Sub(launchCtx.startTime).Milliseconds()
+	if err != nil {
+		log.WithError(err).WithField("cost", installPluginsCost).Error("installing repo plugins: done")
+	} else {
+		log.WithField("cost", installPluginsCost).Info("installing repo plugins: done")
 	}
 
 	// install gitpod plugin
@@ -586,7 +588,7 @@ func resolveUserEnvs() (userEnvs []string, err error) {
 	return
 }
 
-func resolveLaunchContextEnv(configDir string, systemDir string, enableNewUI bool) []string {
+func resolveLaunchContextEnv() []string {
 	var launchCtxEnv []string
 	userEnvs, err := resolveUserEnvs()
 	if err == nil {
@@ -596,21 +598,14 @@ func resolveLaunchContextEnv(configDir string, systemDir string, enableNewUI boo
 		launchCtxEnv = os.Environ()
 	}
 
-	// Set default config and system directories under /workspace to preserve between restarts
-	launchCtxEnv = append(launchCtxEnv,
-		// Set default config and system directories under /workspace to preserve between restarts
-		fmt.Sprintf("IJ_HOST_CONFIG_BASE_DIR=%s", configDir),
-		fmt.Sprintf("IJ_HOST_SYSTEM_BASE_DIR=%s", systemDir),
-	)
-
 	// instead put them into /ide-desktop/${alias}${qualifier}/backend/bin/idea64.vmoptions
 	// otherwise JB will complain to a user on each startup
 	// by default remote dev already set -Xmx2048m, see /ide-desktop/${alias}${qualifier}/backend/plugins/remote-dev-server/bin/launcher.sh
 	launchCtxEnv = append(launchCtxEnv, "JAVA_TOOL_OPTIONS=")
 
-	if enableNewUI {
-		launchCtxEnv = append(launchCtxEnv, "REMOTE_DEV_NEW_UI_ENABLED=1")
-	}
+	// Force it to be disabled as we update platform properties file already
+	// TODO: Some ides have it enabled by default still, check pycharm and remove next release
+	launchCtxEnv = append(launchCtxEnv, "REMOTE_DEV_LEGACY_PER_PROJECT_CONFIGS=0")
 
 	log.WithField("env", strings.Join(launchCtxEnv, "\n")).Info("resolved launch env")
 
@@ -637,6 +632,52 @@ func handleSignal() {
 	log.Info("asked IDE to terminate")
 }
 
+func configurePlatformProperties(platformOptionsPath string, configDir string, systemDir string) (bool, error) {
+	buffer, err := os.ReadFile(platformOptionsPath)
+	if err != nil {
+		return false, err
+	}
+
+	content := string(buffer)
+
+	updated, content := updatePlatformProperties(content, configDir, systemDir)
+
+	if updated {
+		return true, os.WriteFile(platformOptionsPath, []byte(content), 0)
+	}
+
+	return false, nil
+}
+
+func updatePlatformProperties(content string, configDir string, systemDir string) (bool, string) {
+	lines := strings.Split(content, "\n")
+	configMap := make(map[string]bool)
+	for _, v := range lines {
+		v = strings.TrimSpace(v)
+		if v != "" && !strings.HasPrefix(v, "#") {
+			key, _, found := strings.Cut(v, "=")
+			if found {
+				configMap[key] = true
+			}
+		}
+	}
+
+	updated := false
+
+	if _, found := configMap["idea.config.path"]; !found {
+		updated = true
+		content = strings.Join([]string{
+			content,
+			fmt.Sprintf("idea.config.path=%s", configDir),
+			fmt.Sprintf("idea.plugins.path=%s", configDir+"/plugins"),
+			fmt.Sprintf("idea.system.path=%s", systemDir),
+			fmt.Sprintf("idea.log.path=%s", systemDir+"/log"),
+		}, "\n")
+	}
+
+	return updated, content
+}
+
 func configureVMOptions(config *gitpod.GitpodConfig, alias string, vmOptionsPath string) error {
 	options, err := readVMOptions(vmOptionsPath)
 	if err != nil {
@@ -647,7 +688,7 @@ func configureVMOptions(config *gitpod.GitpodConfig, alias string, vmOptionsPath
 }
 
 func readVMOptions(vmOptionsPath string) ([]string, error) {
-	content, err := ioutil.ReadFile(vmOptionsPath)
+	content, err := os.ReadFile(vmOptionsPath)
 	if err != nil {
 		return nil, err
 	}
@@ -657,7 +698,7 @@ func readVMOptions(vmOptionsPath string) ([]string, error) {
 func writeVMOptions(vmOptionsPath string, vmoptions []string) error {
 	// vmoptions file should end with a newline
 	content := strings.Join(vmoptions, "\n") + "\n"
-	return ioutil.WriteFile(vmOptionsPath, []byte(content), 0)
+	return os.WriteFile(vmOptionsPath, []byte(content), 0)
 }
 
 // deduplicateVMOption append new VMOptions onto old VMOptions and remove any duplicated leftmost options
@@ -844,7 +885,7 @@ func syncPlugin(file fs.FileInfo, srcDir, destDir string) error {
 }
 
 func ensureInitialSyncDest(launchCtx *LaunchContext, target SyncTarget) (string, error, bool) {
-	targetDestDir := launchCtx.projectConfigDir
+	targetDestDir := launchCtx.configDir
 	if target == Plugins {
 		targetDestDir = launchCtx.backendDir
 	}
@@ -964,7 +1005,7 @@ func installPlugins(config *gitpod.GitpodConfig, launchCtx *LaunchContext) error
 	installErr := cmd.Run()
 
 	// delete alien_plugins.txt to suppress 3rd-party plugins consent on startup to workaround backend startup freeze
-	err = os.Remove(launchCtx.projectConfigDir + "/alien_plugins.txt")
+	err = os.Remove(launchCtx.configDir + "/alien_plugins.txt")
 	if err != nil && !os.IsNotExist(err) && !strings.Contains(err.Error(), "no such file or directory") {
 		log.WithError(err).Error("failed to suppress 3rd-party plugins consent")
 	}

components/ide/jetbrains/launcher/main_test.go

@@ -75,3 +75,28 @@ func TestUpdateVMOptions(t *testing.T) {
 		})
 	}
 }
+
+func TestUpdatePlatformProperties(t *testing.T) {
+	platformProperties := `#---------------------------------------------------------------------
+	# Uncomment this option if you want to customize a path to the settings directory.
+	#---------------------------------------------------------------------
+	# idea.config.path=${user.home}/.IntelliJIdea/config
+
+
+	shared.indexes.download.auto.consent=true
+	`
+	configDir := "/workspace/.config/JetBrains/RemoteDev-IU"
+	systemDir := "/workspace/.cache/JetBrains/RemoteDev-IU"
+
+	t.Run("updatePlatformProperties multiple time should be stable", func(t *testing.T) {
+
+		updated, content := updatePlatformProperties(platformProperties, configDir, systemDir)
+		assert.Equal(t, true, updated)
+		assert.Equal(t, true, strings.Contains(content, "idea.config.path=/workspace/.config/JetBrains/RemoteDev-IU"))
+		for i := 0; i < 5; i++ {
+			updated, newContent := updatePlatformProperties(content, configDir, systemDir)
+			assert.Equal(t, false, updated)
+			assert.Equal(t, content, newContent)
+		}
+	})
+}

test/tests/ide/jetbrains/gateway_test.go

@@ -21,6 +21,7 @@ import (
 
 	protocol "github.com/gitpod-io/gitpod/gitpod-protocol"
 	supervisor "github.com/gitpod-io/gitpod/supervisor/api"
+	agent "github.com/gitpod-io/gitpod/test/pkg/agent/workspace/api"
 	"github.com/gitpod-io/gitpod/test/pkg/integration"
 	wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
 	"github.com/google/go-github/v42/github"
@@ -150,6 +151,39 @@ func JetBrainsIDETest(ctx context.Context, t *testing.T, cfg *envconf.Config, id
 		t.Fatal(err)
 	}
 
+	if ide == "intellij" {
+		t.Logf("Check idea.log file correct location")
+		rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), integration.WithInstanceID(info.LatestInstance.ID), integration.WithWorkspacekitLift(true))
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer rsa.Close()
+		integration.DeferCloser(t, closer)
+
+		qualifier := ""
+		if useLatest {
+			qualifier = "-latest"
+		}
+
+		var resp agent.ExecResponse
+		err = rsa.Call("WorkspaceAgent.Exec", &agent.ExecRequest{
+			Dir:     "/",
+			Command: "bash",
+			Args: []string{
+				"-c",
+				fmt.Sprintf("test -f /workspace/.cache/JetBrains%s/RemoteDev-IU/log/idea.log", qualifier),
+			},
+		}, &resp)
+
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if resp.ExitCode != 0 {
+			t.Fatal("idea.log file not found in the expected location")
+		}
+	}
+
 	ts := oauth2.StaticTokenSource(
 		&oauth2.Token{AccessToken: roboquatToken},
 	)
@@ -273,7 +307,7 @@ func TestIntellij(t *testing.T) {
 		Assess("it can let JetBrains Gateway connect", func(testCtx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 			ctx, cancel := context.WithTimeout(testCtx, 30*time.Minute)
 			defer cancel()
-			JetBrainsIDETest(ctx, t, cfg, "intellij", "https://github.com/gitpod-samples/spring-petclinic")
+			JetBrainsIDETest(ctx, t, cfg, "intellij", "https://github.com/jeanp413/spring-petclinic")
 			return testCtx
 		}).
 		Feature()