Python client (#1449)

Author: vishalbollu

cli/cmd/delete.go

@@ -45,7 +45,7 @@ func deleteInit() {
 	// only applies to aws provider because local doesn't support multiple replicas
 	_deleteCmd.Flags().BoolVarP(&_flagDeleteForce, "force", "f", false, "delete the api without confirmation")
 	_deleteCmd.Flags().BoolVarP(&_flagDeleteKeepCache, "keep-cache", "c", false, "keep cached data for the api")
-	_deleteCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStrings(), "|")))
+	_deleteCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.UserOutputTypeStrings(), "|")))
 }
 
 var _deleteCmd = &cobra.Command{

cli/cmd/deploy.go

@@ -59,7 +59,7 @@ func deployInit() {
 	_deployCmd.Flags().StringVarP(&_flagDeployEnv, "env", "e", getDefaultEnv(_generalCommandType), "environment to use")
 	_deployCmd.Flags().BoolVarP(&_flagDeployForce, "force", "f", false, "override the in-progress api update")
 	_deployCmd.Flags().BoolVarP(&_flagDeployDisallowPrompt, "yes", "y", false, "skip prompts")
-	_deployCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStrings(), "|")))
+	_deployCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.UserOutputTypeStrings(), "|")))
 }
 
 var _deployCmd = &cobra.Command{
@@ -106,23 +106,31 @@ var _deployCmd = &cobra.Command{
 				exit.Error(err)
 			}
 
+			local.OutputType = _flagOutput // Set output type for the Local package
 			deployResults, err = local.Deploy(env, configPath, projectFiles, _flagDeployDisallowPrompt)
 			if err != nil {
 				exit.Error(err)
 			}
 		}
 
-		if _flagOutput == flags.JSONOutputType {
+		switch _flagOutput {
+		case flags.JSONOutputType:
 			bytes, err := libjson.Marshal(deployResults)
 			if err != nil {
 				exit.Error(err)
 			}
 			fmt.Println(string(bytes))
 			return
+		case flags.MixedOutputType:
+			err := mixedPrint(deployResults)
+			if err != nil {
+				exit.Error(err)
+			}
+			return
+		case flags.PrettyOutputType:
+			message := deployMessage(deployResults, env.Name)
+			print.BoldFirstBlock(message)
 		}
-
-		message := deployMessage(deployResults, env.Name)
-		print.BoldFirstBlock(message)
 	},
 }
 

cli/cmd/env.go

@@ -48,7 +48,7 @@ func envInit() {
 	_envCmd.AddCommand(_envConfigureCmd)
 
 	_envListCmd.Flags().SortFlags = false
-	_envListCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStrings(), "|")))
+	_envListCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.UserOutputTypeStrings(), "|")))
 	_envCmd.AddCommand(_envListCmd)
 
 	_envDefaultCmd.Flags().SortFlags = false

cli/cmd/get.go

@@ -62,7 +62,7 @@ func getInit() {
 	_getCmd.Flags().SortFlags = false
 	_getCmd.Flags().StringVarP(&_flagGetEnv, "env", "e", getDefaultEnv(_generalCommandType), "environment to use")
 	_getCmd.Flags().BoolVarP(&_flagWatch, "watch", "w", false, "re-run the command every 2 seconds")
-	_getCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStrings(), "|")))
+	_getCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.UserOutputTypeStrings(), "|")))
 }
 
 var _getCmd = &cobra.Command{

cli/cmd/refresh.go

@@ -40,7 +40,7 @@ func refreshInit() {
 	_refreshCmd.Flags().SortFlags = false
 	_refreshCmd.Flags().StringVarP(&_flagRefreshEnv, "env", "e", getDefaultEnv(_generalCommandType), "environment to use")
 	_refreshCmd.Flags().BoolVarP(&_flagRefreshForce, "force", "f", false, "override the in-progress api update")
-	_refreshCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStrings(), "|")))
+	_refreshCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.UserOutputTypeStrings(), "|")))
 }
 
 var _refreshCmd = &cobra.Command{

cli/cmd/root.go

@@ -17,6 +17,7 @@ limitations under the License.
 package cmd
 
 import (
+	"encoding/base64"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -25,6 +26,7 @@ import (
 	"github.com/cortexlabs/cortex/cli/types/flags"
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	"github.com/cortexlabs/cortex/pkg/lib/exit"
+	libjson "github.com/cortexlabs/cortex/pkg/lib/json"
 	s "github.com/cortexlabs/cortex/pkg/lib/strings"
 	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
 	homedir "github.com/mitchellh/go-homedir"
@@ -122,11 +124,18 @@ func init() {
 
 func initTelemetry() {
 	cID := clientID()
+
+	invoker := os.Getenv("CORTEX_CLI_INVOKER")
+	if invoker == "" {
+		invoker = "direct"
+	}
+
 	telemetry.Init(telemetry.Config{
 		Enabled: true,
 		UserID:  cID,
 		Properties: map[string]string{
 			"client_id": cID,
+			"invoker":   invoker,
 		},
 		Environment: "cli",
 		LogErrors:   false,
@@ -229,3 +238,12 @@ func printLeadingNewLine() {
 	}
 	fmt.Println("")
 }
+
+func mixedPrint(a interface{}) error {
+	jsonBytes, err := libjson.Marshal(a)
+	if err != nil {
+		return err
+	}
+	fmt.Println(fmt.Sprintf("~~cortex~~%s~~cortex~~", base64.StdEncoding.EncodeToString(jsonBytes)))
+	return nil
+}

cli/local/docker_spec.go

@@ -463,7 +463,7 @@ func retryWithNvidiaRuntime(err error, containerConfig *container.Config, hostCo
 	}
 
 	if _, ok := docker.MustDockerClient().Info.Runtimes["nvidia"]; ok {
-		fmt.Println("retrying API deployment using nvidia runtime because device driver for GPU was not found")
+		localPrintln("retrying API deployment using nvidia runtime because device driver for GPU was not found")
 		hostConfig.Runtime = "nvidia"
 		hostConfig.Resources.DeviceRequests = nil
 		containerCreateRequest, err := docker.MustDockerClient().ContainerCreate(context.Background(), containerConfig, hostConfig, nil, "")

cli/local/model_cache.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cortexlabs/cortex/cli/types/flags"
 	"github.com/cortexlabs/cortex/pkg/lib/archive"
 	"github.com/cortexlabs/cortex/pkg/lib/aws"
 	"github.com/cortexlabs/cortex/pkg/lib/cron"
@@ -75,7 +76,7 @@ func CacheModels(apiSpec *spec.API, awsClient *aws.Client) ([]*spec.LocalModelCa
 	}
 
 	if uncachedModelCount > 0 {
-		fmt.Println("") // Newline to group all of the model information
+		localPrintln("") // Newline to group all of the model information
 	}
 
 	return localModelCaches, nil
@@ -134,13 +135,13 @@ func cacheModel(modelPath string, localModelCache spec.LocalModelCache, awsClien
 				return err
 			}
 		} else if strings.HasSuffix(modelPath, ".onnx") {
-			fmt.Println(fmt.Sprintf("￮ caching model %s ...", modelPath))
+			localPrintln(fmt.Sprintf("￮ caching model %s ...", modelPath))
 			err := files.CopyFileOverwrite(modelPath, filepath.Join(modelDir, filepath.Base(modelPath)))
 			if err != nil {
 				return err
 			}
 		} else {
-			fmt.Println(fmt.Sprintf("￮ caching model %s ...", modelPath))
+			localPrintln(fmt.Sprintf("￮ caching model %s ...", modelPath))
 			tfModelVersion := filepath.Base(modelPath)
 			err := files.CopyDirOverwrite(strings.TrimSuffix(modelPath, "/"), s.EnsureSuffix(filepath.Join(modelDir, tfModelVersion), "/"))
 			if err != nil {
@@ -196,10 +197,12 @@ func DeleteCachedModelsByID(modelIDs []string) error {
 }
 
 func downloadModel(modelPath string, modelDir string, awsClientForBucket *aws.Client) error {
-	fmt.Printf("￮ downloading model %s ", modelPath)
-	defer fmt.Print(" ✓\n")
-	dotCron := cron.Run(print.Dot, nil, 2*time.Second)
-	defer dotCron.Cancel()
+	localPrintf("￮ downloading model %s ", modelPath)
+	defer localPrint(" ✓\n")
+	if OutputType != flags.JSONOutputType {
+		dotCron := cron.Run(print.Dot, nil, 2*time.Second)
+		defer dotCron.Cancel()
+	}
 
 	bucket, prefix, err := aws.SplitS3Path(modelPath)
 	if err != nil {
@@ -234,7 +237,7 @@ func downloadModel(modelPath string, modelDir string, awsClientForBucket *aws.Cl
 }
 
 func unzipAndValidate(originalModelPath string, zipFile string, destPath string) error {
-	fmt.Println(fmt.Sprintf("￮ unzipping model %s ...", originalModelPath))
+	localPrintln(fmt.Sprintf("￮ unzipping model %s ...", originalModelPath))
 	tmpDir := filepath.Join(filepath.Dir(destPath), filepath.Base(destPath)+"-tmp")
 	err := files.CreateDir(tmpDir)
 	if err != nil {

cli/local/print.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2020 Cortex Labs, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package local
+
+import (
+	"fmt"
+
+	"github.com/cortexlabs/cortex/cli/types/flags"
+)
+
+// Can be overwritten by CLI commands
+var OutputType flags.OutputType = flags.PrettyOutputType
+
+func localPrintln(a ...interface{}) {
+	if OutputType != flags.JSONOutputType {
+		fmt.Println(a...)
+	}
+}
+
+func localPrint(a ...interface{}) {
+	if OutputType != flags.JSONOutputType {
+		fmt.Print(a...)
+	}
+}
+
+func localPrintf(format string, a ...interface{}) {
+	if OutputType != flags.JSONOutputType {
+		fmt.Printf(format, a...)
+	}
+}

cli/local/validations.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/cortexlabs/cortex/cli/types/flags"
 	"github.com/cortexlabs/cortex/pkg/lib/aws"
 	"github.com/cortexlabs/cortex/pkg/lib/docker"
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
@@ -151,7 +152,12 @@ func ValidateLocalAPIs(apis []userconfig.API, projectFiles ProjectFiles, awsClie
 			}
 		}
 
-		pulledThisImage, err := docker.PullImage(image, dockerAuth, docker.PrintDots)
+		pullVerbosity := docker.PrintDots
+		if OutputType == flags.JSONOutputType {
+			pullVerbosity = docker.NoPrint
+		}
+
+		pulledThisImage, err := docker.PullImage(image, dockerAuth, pullVerbosity)
 		if err != nil {
 			return errors.Wrap(err, "failed to pull image", image)
 		}
@@ -162,7 +168,7 @@ func ValidateLocalAPIs(apis []userconfig.API, projectFiles ProjectFiles, awsClie
 	}
 
 	if pulledImage {
-		fmt.Println()
+		localPrintln()
 	}
 
 	portToRunningAPIsMap, err := getPortToRunningAPIsMap()