Avoid docker mounting to support CLI commands inside of a docker container (#1370)

Author: deliahu

cli/cluster/lib_http_client.go

@@ -27,10 +27,10 @@ import (
 	"time"
 
 	"github.com/cortexlabs/cortex/pkg/consts"
+	"github.com/cortexlabs/cortex/pkg/lib/archive"
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	"github.com/cortexlabs/cortex/pkg/lib/files"
 	"github.com/cortexlabs/cortex/pkg/lib/json"
-	"github.com/cortexlabs/cortex/pkg/lib/zip"
 	"github.com/cortexlabs/cortex/pkg/operator/schema"
 )
 
@@ -154,8 +154,8 @@ func addFileToMultipart(fileName string, writer *multipart.Writer, reader io.Rea
 	return nil
 }
 
-func HTTPUploadZip(operatorConfig OperatorConfig, endpoint string, zipInput *zip.Input, fileName string, qParams ...map[string]string) ([]byte, error) {
-	zipBytes, err := zip.ToMem(zipInput)
+func HTTPUploadZip(operatorConfig OperatorConfig, endpoint string, zipInput *archive.Input, fileName string, qParams ...map[string]string) ([]byte, error) {
+	zipBytes, _, err := archive.ZipToMem(zipInput)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to zip configuration file")
 	}

cli/cmd/cluster.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -158,7 +159,7 @@ var _upCmd = &cobra.Command{
 			}
 		}
 
-		out, exitCode, err := runManagerWithClusterConfig("/root/install.sh", clusterConfig, awsCreds, _flagClusterEnv)
+		out, exitCode, err := runManagerWithClusterConfig("/root/install.sh", clusterConfig, awsCreds, _flagClusterEnv, nil, nil)
 		if err != nil {
 			if clusterConfig.APIGatewaySetting == clusterconfig.PublicAPIGatewaySetting {
 				awsClient.DeleteAPIGatewayByTag(clusterconfig.ClusterNameTag, clusterConfig.ClusterName) // best effort deletion
@@ -238,6 +239,30 @@ var _upCmd = &cobra.Command{
 			exit.Error(ErrorClusterUp(out + helpStr))
 		}
 
+		loadBalancer, err := awsClient.FindLoadBalancer(map[string]string{
+			clusterconfig.ClusterNameTag: clusterConfig.ClusterName,
+			"cortex.dev/load-balancer":   "operator",
+		})
+		if err != nil {
+			exit.Error(errors.Append(err, fmt.Sprintf("\n\nunable to locate operator load balancer; you can attempt to resolve this issue and configure your CLI environment by running `cortex cluster info --env %s`", _flagClusterEnv)))
+		}
+		if loadBalancer == nil {
+			exit.Error(ErrorNoOperatorLoadBalancer(_flagClusterEnv))
+		}
+
+		newEnvironment := cliconfig.Environment{
+			Name:               _flagClusterEnv,
+			Provider:           types.AWSProviderType,
+			OperatorEndpoint:   pointer.String("https://" + *loadBalancer.DNSName),
+			AWSAccessKeyID:     pointer.String(awsCreds.CortexAWSAccessKeyID),
+			AWSSecretAccessKey: pointer.String(awsCreds.CortexAWSSecretAccessKey),
+		}
+
+		err = addEnvToCLIConfig(newEnvironment)
+		if err != nil {
+			exit.Error(errors.Append(err, fmt.Sprintf("unable to configure cli environment; you can attempt to resolve this issue and configure your CLI environment by running `cortex cluster info --env %s`", _flagClusterEnv)))
+		}
+
 		fmt.Printf(console.Bold("\nan environment named \"%s\" has been configured for this cluster; append `--env %s` to cortex commands to connect to it (e.g. `cortex deploy --env %s`), or set it as your default with `cortex env default %s`\n"), _flagClusterEnv, _flagClusterEnv, _flagClusterEnv, _flagClusterEnv)
 	},
 }
@@ -289,7 +314,7 @@ var _configureCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		out, exitCode, err := runManagerWithClusterConfig("/root/install.sh --update", clusterConfig, awsCreds, _flagClusterEnv)
+		out, exitCode, err := runManagerWithClusterConfig("/root/install.sh --update", clusterConfig, awsCreds, _flagClusterEnv, nil, nil)
 		if err != nil {
 			exit.Error(err)
 		}
@@ -375,6 +400,12 @@ var _downCmd = &cobra.Command{
 			}
 		}
 
+		// updating CLI env is best-effort, so ignore errors
+		loadBalancer, _ := awsClient.FindLoadBalancer(map[string]string{
+			clusterconfig.ClusterNameTag: *accessConfig.ClusterName,
+			"cortex.dev/load-balancer":   "operator",
+		})
+
 		if !_flagClusterDisallowPrompt {
 			prompt.YesOrExit(fmt.Sprintf("your cluster named \"%s\" in %s will be spun down and all apis will be deleted, are you sure you want to continue?", *accessConfig.ClusterName, *accessConfig.Region), "", "")
 		}
@@ -421,7 +452,7 @@ var _downCmd = &cobra.Command{
 		}
 
 		fmt.Println("￮ spinning down the cluster ...")
-		out, exitCode, err := runManagerAccessCommand("/root/uninstall.sh", *accessConfig, awsCreds, _flagClusterEnv)
+		out, exitCode, err := runManagerAccessCommand("/root/uninstall.sh", *accessConfig, awsCreds, _flagClusterEnv, nil, nil)
 		if err != nil {
 			exit.Error(err)
 		}
@@ -431,8 +462,24 @@ var _downCmd = &cobra.Command{
 			exit.Error(ErrorClusterDown(out + helpStr))
 		}
 
-		cachedConfigPath := cachedClusterConfigPath(*accessConfig.ClusterName, *accessConfig.Region)
-		os.Remove(cachedConfigPath)
+		// best-effort deletion of cli environment(s)
+		if loadBalancer != nil {
+			envNames, isDefaultEnv, _ := getEnvNamesByOperatorEndpoint(*loadBalancer.DNSName)
+			if len(envNames) > 0 {
+				for _, envName := range envNames {
+					removeEnvFromCLIConfig(envName)
+				}
+				fmt.Printf("✓ deleted the %s environment configuration%s\n", s.StrsAnd(envNames), s.SIfPlural(len(envNames)))
+				if isDefaultEnv {
+					fmt.Println("✓ set the default environment to local")
+				}
+			}
+		}
+
+		fmt.Printf("\nplease check CloudFormation to ensure that all resources for the %s cluster eventually become successfully deleted: %s\n", *accessConfig.ClusterName, clusterstate.CloudFormationURL(*accessConfig.ClusterName, *accessConfig.Region))
+
+		cachedClusterConfigPath := cachedClusterConfigPath(*accessConfig.ClusterName, *accessConfig.Region)
+		os.Remove(cachedClusterConfigPath)
 	},
 }
 
@@ -491,7 +538,7 @@ func cmdInfo(awsCreds AWSCredentials, accessConfig *clusterconfig.AccessConfig,
 
 	clusterConfig := refreshCachedClusterConfig(awsCreds, accessConfig, disallowPrompt)
 
-	out, exitCode, err := runManagerWithClusterConfig("/root/info.sh", &clusterConfig, awsCreds, _flagClusterEnv)
+	out, exitCode, err := runManagerWithClusterConfig("/root/info.sh", &clusterConfig, awsCreds, _flagClusterEnv, nil, nil)
 	if err != nil {
 		exit.Error(err)
 	}
@@ -724,36 +771,43 @@ func updateInfoEnvironment(operatorEndpoint string, awsCreds AWSCredentials, dis
 }
 
 func cmdDebug(awsCreds AWSCredentials, accessConfig *clusterconfig.AccessConfig) {
-	out, exitCode, err := runManagerAccessCommand("/root/debug.sh", *accessConfig, awsCreds, _flagClusterEnv)
+	// note: if modifying this string, also change it in files.IgnoreCortexDebug()
+	debugFileName := fmt.Sprintf("cortex-debug-%s.tgz", time.Now().UTC().Format("2006-01-02-15-04-05"))
+
+	containerDebugPath := "/out/" + debugFileName
+	copyFromPaths := []dockerCopyFromPath{
+		{
+			containerPath: containerDebugPath,
+			localDir:      _cwd,
+		},
+	}
+
+	out, exitCode, err := runManagerAccessCommand("/root/debug.sh "+containerDebugPath, *accessConfig, awsCreds, _flagClusterEnv, nil, copyFromPaths)
 	if err != nil {
 		exit.Error(err)
 	}
 	if exitCode == nil || *exitCode != 0 {
 		exit.Error(ErrorClusterDebug(out))
 	}
 
-	timestamp := time.Now().UTC().Format("2006-01-02-15-04-05")
-	userDebugPath := fmt.Sprintf("cortex-debug-%s.tgz", timestamp) // note: if modifying this string, also change it in files.IgnoreCortexDebug()
-	err = os.Rename(_debugPath, userDebugPath)
-	if err != nil {
-		exit.Error(errors.WithStack(err))
-	}
-
-	fmt.Println("saved cluster info to ./" + userDebugPath)
+	fmt.Println("saved cluster info to ./" + debugFileName)
 	return
 }
 
 func refreshCachedClusterConfig(awsCreds AWSCredentials, accessConfig *clusterconfig.AccessConfig, disallowPrompt bool) clusterconfig.Config {
 	// add empty file if cached cluster doesn't exist so that the file output by manager container maintains current user permissions
-	cachedConfigPath := cachedClusterConfigPath(*accessConfig.ClusterName, *accessConfig.Region)
-	if !files.IsFile(cachedConfigPath) {
-		files.MakeEmptyFile(cachedConfigPath)
-	}
+	cachedClusterConfigPath := cachedClusterConfigPath(*accessConfig.ClusterName, *accessConfig.Region)
+	containerConfigPath := fmt.Sprintf("/out/%s", filepath.Base(cachedClusterConfigPath))
 
-	mountedConfigPath := mountedClusterConfigPath(*accessConfig.ClusterName, *accessConfig.Region)
+	copyFromPaths := []dockerCopyFromPath{
+		{
+			containerPath: containerConfigPath,
+			localDir:      files.Dir(cachedClusterConfigPath),
+		},
+	}
 
 	fmt.Print("syncing cluster configuration ...\n\n")
-	out, exitCode, err := runManagerAccessCommand("/root/refresh.sh "+mountedConfigPath, *accessConfig, awsCreds, _flagClusterEnv)
+	out, exitCode, err := runManagerAccessCommand("/root/refresh.sh "+containerConfigPath, *accessConfig, awsCreds, _flagClusterEnv, nil, copyFromPaths)
 	if err != nil {
 		exit.Error(err)
 	}
@@ -762,7 +816,7 @@ func refreshCachedClusterConfig(awsCreds AWSCredentials, accessConfig *clusterco
 	}
 
 	refreshedClusterConfig := &clusterconfig.Config{}
-	readCachedClusterConfigFile(refreshedClusterConfig, cachedConfigPath)
+	readCachedClusterConfigFile(refreshedClusterConfig, cachedClusterConfigPath)
 	return *refreshedClusterConfig
 }
 

cli/cmd/deploy.go

@@ -24,6 +24,7 @@ import (
 	"github.com/cortexlabs/cortex/cli/cluster"
 	"github.com/cortexlabs/cortex/cli/local"
 	"github.com/cortexlabs/cortex/cli/types/flags"
+	"github.com/cortexlabs/cortex/pkg/lib/archive"
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	"github.com/cortexlabs/cortex/pkg/lib/exit"
 	"github.com/cortexlabs/cortex/pkg/lib/files"
@@ -34,7 +35,6 @@ import (
 	s "github.com/cortexlabs/cortex/pkg/lib/strings"
 	"github.com/cortexlabs/cortex/pkg/lib/table"
 	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
-	"github.com/cortexlabs/cortex/pkg/lib/zip"
 	"github.com/cortexlabs/cortex/pkg/operator/schema"
 	"github.com/cortexlabs/cortex/pkg/types"
 	"github.com/cortexlabs/cortex/pkg/types/userconfig"
@@ -219,8 +219,8 @@ func getDeploymentBytes(provider types.ProviderType, configPath string) (map[str
 		didPromptFileCount = true
 	}
 
-	projectZipBytes, err := zip.ToMem(&zip.Input{
-		FileLists: []zip.FileListInput{
+	projectZipBytes, _, err := archive.ZipToMem(&archive.Input{
+		FileLists: []archive.FileListInput{
 			{
 				Sources:      projectPaths,
 				RemovePrefix: projectRoot,

cli/cmd/errors.go

@@ -49,6 +49,7 @@ const (
 	ErrOperatorConfigFromLocalEnvironment   = "cli.operater_config_from_local_environment"
 	ErrFieldNotFoundInEnvironment           = "cli.field_not_found_in_environment"
 	ErrInvalidOperatorEndpoint              = "cli.invalid_operator_endpoint"
+	ErrNoOperatorLoadBalancer               = "cli.no_operator_load_balancer"
 	ErrCortexYAMLNotFound                   = "cli.cortex_yaml_not_found"
 	ErrConnectToDockerDaemon                = "cli.connect_to_docker_daemon"
 	ErrDockerPermissions                    = "cli.docker_permissions"
@@ -130,6 +131,14 @@ func ErrorInvalidOperatorEndpoint(endpoint string) error {
 	})
 }
 
+// err can be passed in as nil
+func ErrorNoOperatorLoadBalancer(envName string) error {
+	return errors.WithStack(&errors.Error{
+		Kind:    ErrNoOperatorLoadBalancer,
+		Message: fmt.Sprintf("unable to locate operator load balancer; you can attempt to resolve this issue and configure your CLI environment by running `cortex cluster info --env %s`", envName),
+	})
+}
+
 func ErrorCortexYAMLNotFound() error {
 	return errors.WithStack(&errors.Error{
 		Kind:    ErrCortexYAMLNotFound,

cli/cmd/lib_cli_config.go

@@ -890,6 +890,28 @@ func removeEnvFromCLIConfig(envName string) error {
 	return nil
 }
 
+// returns the list of environment names, and whether any of them were the default
+func getEnvNamesByOperatorEndpoint(operatorEndpoint string) ([]string, bool, error) {
+	cliConfig, err := readCLIConfig()
+	if err != nil {
+		return nil, false, err
+	}
+
+	var envNames []string
+	isDefaultEnv := false
+
+	for _, env := range cliConfig.Environments {
+		if env.OperatorEndpoint != nil && s.LastSplit(*env.OperatorEndpoint, "//") == s.LastSplit(operatorEndpoint, "//") {
+			envNames = append(envNames, env.Name)
+			if env.Name == cliConfig.DefaultEnvironment {
+				isDefaultEnv = true
+			}
+		}
+	}
+
+	return envNames, isDefaultEnv, nil
+}
+
 func readCLIConfig() (cliconfig.CLIConfig, error) {
 	if _cachedCLIConfig != nil {
 		return *_cachedCLIConfig, nil

cli/cmd/lib_cluster_config.go

@@ -42,10 +42,6 @@ func cachedClusterConfigPath(clusterName string, region string) string {
 	return filepath.Join(_localDir, fmt.Sprintf("cluster_%s_%s.yaml", clusterName, region))
 }
 
-func mountedClusterConfigPath(clusterName string, region string) string {
-	return filepath.Join("/.cortex", fmt.Sprintf("cluster_%s_%s.yaml", clusterName, region))
-}
-
 func existingCachedClusterConfigPaths() []string {
 	paths, err := files.ListDir(_localDir, false)
 	if err != nil {