Add prompts for zip file count and total size (#767)

Author: deliahu

cli/cmd/cluster.go

@@ -231,7 +231,7 @@ var _downCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		prompt.YesOrExit(fmt.Sprintf("your cluster (%s in %s) will be spun down and all apis will be deleted, are you sure you want to continue?", *accessConfig.ClusterName, *accessConfig.Region), "")
+		prompt.YesOrExit(fmt.Sprintf("your cluster (%s in %s) will be spun down and all apis will be deleted, are you sure you want to continue?", *accessConfig.ClusterName, *accessConfig.Region), "", "")
 
 		out, exitCode, err := runManagerAccessCommand("/root/uninstall.sh", *accessConfig, awsCreds)
 		if err != nil {

cli/cmd/delete.go

@@ -54,7 +54,7 @@ func delete(apiName string, keepCache bool) {
 	if !_flagDeleteForce {
 		readyReplicas := getReadyReplicasOrNil(apiName)
 		if readyReplicas != nil && *readyReplicas > 2 {
-			prompt.YesOrExit(fmt.Sprintf("are you sure you want to delete the %s api (which has %d running replicas)?", apiName, *readyReplicas), "")
+			prompt.YesOrExit(fmt.Sprintf("are you sure you want to delete the %s api (which has %d running replicas)?", apiName, *readyReplicas), "", "")
 		}
 	}
 

cli/cmd/deploy.go

@@ -27,18 +27,24 @@ import (
 	"github.com/cortexlabs/cortex/pkg/lib/exit"
 	"github.com/cortexlabs/cortex/pkg/lib/files"
 	"github.com/cortexlabs/cortex/pkg/lib/json"
+	"github.com/cortexlabs/cortex/pkg/lib/prompt"
 	s "github.com/cortexlabs/cortex/pkg/lib/strings"
 	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
 	"github.com/cortexlabs/cortex/pkg/lib/zip"
 	"github.com/cortexlabs/cortex/pkg/operator/schema"
 	"github.com/spf13/cobra"
 )
 
-var _warningFileSize = 1024 * 1024 * 10
+var _warningFileBytes = 1024 * 1024 * 10
+var _warningProjectBytes = 1024 * 1024 * 10
+var _warningFileCount = 100
+
 var _flagDeployForce bool
+var _flagDeployYes bool
 
 func init() {
 	_deployCmd.PersistentFlags().BoolVarP(&_flagDeployForce, "force", "f", false, "override the in-progress api update")
+	_deployCmd.PersistentFlags().BoolVarP(&_flagDeployYes, "yes", "y", false, "skip prompts")
 	addEnvFlag(_deployCmd)
 }
 
@@ -89,18 +95,35 @@ func deploy(configPath string, force bool) {
 
 	projectRoot := filepath.Dir(files.UserRelToAbsPath(configPath))
 
-	projectPaths, err := files.ListDirRecursive(projectRoot, false,
+	ignoreFns := []files.IgnoreFn{
 		files.IgnoreSpecificFiles(files.UserRelToAbsPath(configPath)),
 		files.IgnoreCortexDebug,
 		files.IgnoreHiddenFiles,
 		files.IgnoreHiddenFolders,
 		files.IgnorePythonGeneratedFiles,
-		files.PromptForFilesAboveSize(_warningFileSize),
-	)
+	}
+	if !_flagDeployYes {
+		ignoreFns = append(ignoreFns, files.PromptForFilesAboveSize(_warningFileBytes, "do you want to upload %s (%s)?"))
+	}
+
+	projectPaths, err := files.ListDirRecursive(projectRoot, false, ignoreFns...)
 	if err != nil {
 		exit.Error(err)
 	}
 
+	canSkipPromptMsg := "you can skip this prompt next time with `cortex deploy --yes`\n"
+	rootDirMsg := "this directory"
+	if s.EnsureSuffix(projectRoot, "/") != _cwd {
+		rootDirMsg = fmt.Sprintf("./%s", files.DirPathRelativeToCWD(projectRoot))
+	}
+
+	didPromptFileCount := false
+	if !_flagDeployYes && len(projectPaths) >= _warningFileCount {
+		msg := fmt.Sprintf("cortex will zip %d files in %s and upload them to the cluster, though we recommend you upload large files (e.g. models) to s3 and download them in your api's __init__ function. Would you like to continue?", len(projectPaths), rootDirMsg)
+		prompt.YesOrExit(msg, canSkipPromptMsg, "")
+		didPromptFileCount = true
+	}
+
 	projectZipBytes, err := zip.ToMem(&zip.Input{
 		FileLists: []zip.FileListInput{
 			{
@@ -109,11 +132,15 @@ func deploy(configPath string, force bool) {
 			},
 		},
 	})
-
 	if err != nil {
 		exit.Error(errors.Wrap(err, "failed to zip project folder"))
 	}
 
+	if !_flagDeployYes && !didPromptFileCount && len(projectZipBytes) >= _warningProjectBytes {
+		msg := fmt.Sprintf("cortex will zip %d files in %s (%s) and upload them to the cluster, though we recommend you upload large files (e.g. models) to s3 and download them in your api's __init__ function. Would you like to continue?", len(projectPaths), rootDirMsg, s.IntToBase2Byte(len(projectZipBytes)))
+		prompt.YesOrExit(msg, canSkipPromptMsg, "")
+	}
+
 	uploadBytes["project.zip"] = projectZipBytes
 
 	uploadInput := &HTTPUploadInput{

cli/cmd/lib_cluster_config.go

@@ -335,14 +335,14 @@ func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsCreds A
 	}
 
 	exitMessage := fmt.Sprintf("cluster configuration can be modified via the cluster config file; see https://cortex.dev/v/%s/cluster-management/config for more information", consts.CortexVersionMinor)
-	prompt.YesOrExit("would you like to continue?", exitMessage)
+	prompt.YesOrExit("would you like to continue?", "", exitMessage)
 }
 
 func confirmUpdateClusterConfig(clusterConfig clusterconfig.Config, awsCreds AWSCredentials, awsClient *aws.Client) {
 	fmt.Println(clusterConfigConfirmaionStr(clusterConfig, awsCreds, awsClient))
 
 	exitMessage := fmt.Sprintf("cluster configuration can be modified via the cluster config file; see https://cortex.dev/v/%s/cluster-management/config for more information", consts.CortexVersionMinor)
-	prompt.YesOrExit(fmt.Sprintf("your cluster (%s in %s) will be updated according to the configuration above, are you sure you want to continue?", clusterConfig.ClusterName, *clusterConfig.Region), exitMessage)
+	prompt.YesOrExit(fmt.Sprintf("your cluster (%s in %s) will be updated according to the configuration above, are you sure you want to continue?", clusterConfig.ClusterName, *clusterConfig.Region), "", exitMessage)
 }
 
 func clusterConfigConfirmaionStr(clusterConfig clusterconfig.Config, awsCreds AWSCredentials, awsClient *aws.Client) string {

cli/cmd/lib_manager.go

@@ -182,7 +182,7 @@ func runManager(containerConfig *container.Config) (string, *int, error) {
 
 	output := outputBuffer.String()
 
-	// Let the ctrl+C handler run its course
+	// Let the ctrl+c handler run its course
 	if caughtCtrlC {
 		time.Sleep(5 * time.Second)
 	}

cli/cmd/root.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 
 	"github.com/cortexlabs/cortex/pkg/lib/exit"
+	s "github.com/cortexlabs/cortex/pkg/lib/strings"
 	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/spf13/cobra"
@@ -37,10 +38,17 @@ var _cliConfigPath string
 var _clientIDPath string
 var _emailPath string
 var _debugPath string
+var _cwd string
 
 var _flagEnv string
 
 func init() {
+	cwd, err := os.Getwd()
+	if err != nil {
+		exit.Error(err)
+	}
+	_cwd = s.EnsureSuffix(cwd, "/")
+
 	homeDir, err := homedir.Dir()
 	if err != nil {
 		exit.Error(err)

docs/cluster-management/cli.md

@@ -14,6 +14,7 @@ Flags:
   -e, --env string   environment (default "default")
   -f, --force        override the in-progress api update
   -h, --help         help for deploy
+  -y, --yes          skip prompts
 ```
 
 ## get

docs/contributing/development.md

@@ -150,7 +150,7 @@ If you're making changes in the operator and want faster iterations, you can run
 
 If you want to switch back to the in-cluster operator:
 
-1. `<ctrl+C>` to stop your off-cluster operator
+1. `<ctrl+c>` to stop your off-cluster operator
 2. `make operator-start` to install the operator in your cluster
 
 ## Dev workflow

pkg/lib/files/files.go

@@ -127,6 +127,10 @@ func PathRelativeToCWD(absPath string) string {
 	return strings.TrimPrefix(absPath, cwd)
 }
 
+func DirPathRelativeToCWD(absPath string) string {
+	return s.EnsureSuffix(PathRelativeToCWD(absPath), "/")
+}
+
 func IsFileOrDir(path string) bool {
 	_, err := os.Stat(path)
 	if err == nil {
@@ -343,11 +347,16 @@ func IgnoreSpecificFiles(absPaths ...string) IgnoreFn {
 	}
 }
 
-func PromptForFilesAboveSize(size int) IgnoreFn {
+// promptMsgTemplate should have two placeholders: the first is for the file path and the second is for the file size
+func PromptForFilesAboveSize(size int, promptMsgTemplate string) IgnoreFn {
+	if promptMsgTemplate == "" {
+		promptMsgTemplate = "do you want to zip %s (%s)?"
+	}
+
 	return func(path string, fi os.FileInfo) (bool, error) {
 		if !fi.IsDir() && fi.Size() > int64(size) {
-			promptMsg := fmt.Sprintf("do you want to zip %s (%s)?", PathRelativeToCWD(path), s.IntToBase2Byte(int(fi.Size())))
-			return !prompt.YesOrNo(promptMsg, true), nil
+			promptMsg := fmt.Sprintf(promptMsgTemplate, PathRelativeToCWD(path), s.IntToBase2Byte(int(fi.Size())))
+			return !prompt.YesOrNo(promptMsg, "", ""), nil
 		}
 		return false, nil
 	}

pkg/lib/prompt/prompt.go

@@ -74,20 +74,23 @@ func Prompt(opts *Options) string {
 	return val
 }
 
-func YesOrExit(prompt string, exitMessage string) {
+func YesOrExit(prompt string, yesMessage string, noMessage string) {
 	for true {
 		str := Prompt(&Options{
 			Prompt:      prompt + " (y/n)",
 			HideDefault: true,
 		})
 
 		if strings.ToLower(str) == "y" {
+			if yesMessage != "" {
+				fmt.Println(yesMessage)
+			}
 			return
 		}
 
 		if strings.ToLower(str) == "n" {
-			if exitMessage != "" {
-				fmt.Println(exitMessage)
+			if noMessage != "" {
+				fmt.Println(noMessage)
 			}
 			exit.ErrorNoPrintNoTelemetry()
 		}
@@ -97,25 +100,24 @@ func YesOrExit(prompt string, exitMessage string) {
 	}
 }
 
-func YesOrNo(prompt string, showCancelHint bool) bool {
+func YesOrNo(prompt string, yesMessage string, noMessage string) bool {
 	for true {
-		var fullPrompt string
-		if showCancelHint {
-			fullPrompt = prompt + " (y/n, ctrl+C to cancel)"
-		} else {
-			fullPrompt = prompt + " (y/n)"
-		}
-
 		str := Prompt(&Options{
-			Prompt:      fullPrompt,
+			Prompt:      prompt + " (y/n)",
 			HideDefault: true,
 		})
 
 		if strings.ToLower(str) == "y" {
+			if yesMessage != "" {
+				fmt.Println(yesMessage)
+			}
 			return true
 		}
 
 		if strings.ToLower(str) == "n" {
+			if noMessage != "" {
+				fmt.Println(noMessage)
+			}
 			return false
 		}