Cortex Patch (#1651)

Author: vishalbollu

cli/cluster/patch.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2020 Cortex Labs, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"path/filepath"
+
+	"github.com/cortexlabs/cortex/pkg/lib/errors"
+	"github.com/cortexlabs/cortex/pkg/lib/files"
+	"github.com/cortexlabs/cortex/pkg/lib/json"
+	s "github.com/cortexlabs/cortex/pkg/lib/strings"
+	"github.com/cortexlabs/cortex/pkg/operator/schema"
+)
+
+func Patch(operatorConfig OperatorConfig, configPath string, force bool) ([]schema.DeployResult, error) {
+	params := map[string]string{
+		"force":          s.Bool(force),
+		"configFileName": filepath.Base(configPath),
+	}
+
+	configBytes, err := files.ReadFileBytes(configPath)
+	if err != nil {
+		return nil, err
+	}
+
+	response, err := HTTPPostJSON(operatorConfig, "/patch", configBytes, params)
+	if err != nil {
+		return nil, err
+	}
+
+	var deployResults []schema.DeployResult
+	if err := json.Unmarshal(response, &deployResults); err != nil {
+		return nil, errors.Wrap(err, "/patch", string(response))
+	}
+
+	return deployResults, nil
+}

cli/cmd/cluster.go

@@ -46,6 +46,7 @@ import (
 	"github.com/cortexlabs/cortex/pkg/types/clusterconfig"
 	"github.com/cortexlabs/cortex/pkg/types/clusterstate"
 	"github.com/cortexlabs/cortex/pkg/types/userconfig"
+	"github.com/cortexlabs/yaml"
 	"github.com/spf13/cobra"
 )
 
@@ -687,7 +688,12 @@ var _clusterExportCmd = &cobra.Command{
 				exit.Error(err)
 			}
 
-			err = awsClient.DownloadFileFromS3(info.ClusterConfig.Bucket, apiResponse.Spec.RawAPIKey(info.ClusterConfig.ClusterName), path.Join(baseDir, apiResponse.Spec.FileName))
+			yamlBytes, err := yaml.Marshal(apiResponse.Spec.API.SubmittedAPISpec)
+			if err != nil {
+				exit.Error(err)
+			}
+
+			err = files.WriteFile(yamlBytes, path.Join(baseDir, apiResponse.Spec.FileName))
 			if err != nil {
 				exit.Error(err)
 			}

cli/cmd/deploy.go

@@ -195,6 +195,7 @@ func findProjectFiles(provider types.ProviderType, configPath string) ([]string,
 		return nil, err
 	}
 
+	// Include .env file containing environment variables
 	dotEnvPath := path.Join(projectRoot, ".env")
 	if files.IsFile(dotEnvPath) {
 		projectPaths = append(projectPaths, dotEnvPath)

cli/cmd/patch.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2020 Cortex Labs, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/cortexlabs/cortex/cli/cluster"
+	"github.com/cortexlabs/cortex/cli/local"
+	"github.com/cortexlabs/cortex/cli/types/flags"
+	"github.com/cortexlabs/cortex/pkg/lib/exit"
+	libjson "github.com/cortexlabs/cortex/pkg/lib/json"
+	"github.com/cortexlabs/cortex/pkg/lib/print"
+	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
+	"github.com/cortexlabs/cortex/pkg/operator/schema"
+	"github.com/cortexlabs/cortex/pkg/types"
+	"github.com/spf13/cobra"
+)
+
+var (
+	_flagPatchEnv   string
+	_flagPatchForce bool
+)
+
+func patchInit() {
+	_patchCmd.Flags().SortFlags = false
+	_patchCmd.Flags().StringVarP(&_flagPatchEnv, "env", "e", getDefaultEnv(_generalCommandType), "environment to use")
+	_patchCmd.Flags().BoolVarP(&_flagPatchForce, "force", "f", false, "override the in-progress api update")
+	_patchCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.UserOutputTypeStrings(), "|")))
+}
+
+var _patchCmd = &cobra.Command{
+	Use:   "patch [CONFIG_FILE]",
+	Short: "update API configuration for a deployed API",
+	Args:  cobra.RangeArgs(0, 1),
+	Run: func(cmd *cobra.Command, args []string) {
+		env, err := ReadOrConfigureEnv(_flagPatchEnv)
+		if err != nil {
+			telemetry.Event("cli.patch")
+			exit.Error(err)
+		}
+		telemetry.Event("cli.patch", map[string]interface{}{"provider": env.Provider.String(), "env_name": env.Name})
+
+		err = printEnvIfNotSpecified(_flagPatchEnv, cmd)
+		if err != nil {
+			exit.Error(err)
+		}
+
+		configPath := getConfigPath(args)
+
+		var deployResults []schema.DeployResult
+		if env.Provider == types.AWSProviderType {
+			deployResults, err = cluster.Patch(MustGetOperatorConfig(env.Name), configPath, _flagPatchForce)
+			if err != nil {
+				exit.Error(err)
+			}
+		} else {
+			deployResults, err = local.Patch(env, configPath)
+			if err != nil {
+				exit.Error(err)
+			}
+		}
+
+		switch _flagOutput {
+		case flags.JSONOutputType:
+			bytes, err := libjson.Marshal(deployResults)
+			if err != nil {
+				exit.Error(err)
+			}
+			fmt.Println(string(bytes))
+		case flags.PrettyOutputType:
+			message := deployMessage(deployResults, env.Name)
+			if didAnyResultsError(deployResults) {
+				print.StderrBoldFirstBlock(message)
+			} else {
+				print.BoldFirstBlock(message)
+			}
+		}
+
+		if didAnyResultsError(deployResults) {
+			exit.Error(nil)
+		}
+	},
+}

cli/cmd/root.go

@@ -127,6 +127,7 @@ func init() {
 	envInit()
 	getInit()
 	logsInit()
+	patchInit()
 	predictInit()
 	refreshInit()
 	versionInit()
@@ -166,6 +167,7 @@ func Execute() {
 
 	_rootCmd.AddCommand(_deployCmd)
 	_rootCmd.AddCommand(_getCmd)
+	_rootCmd.AddCommand(_patchCmd)
 	_rootCmd.AddCommand(_logsCmd)
 	_rootCmd.AddCommand(_refreshCmd)
 	_rootCmd.AddCommand(_predictCmd)

cli/local/api.go

@@ -37,7 +37,7 @@ import (
 
 var _deploymentID = "local"
 
-func UpdateAPI(apiConfig *userconfig.API, models []spec.CuratedModelResource, configPath string, projectID string, deployDisallowPrompt bool, awsClient *aws.Client) (*schema.APIResponse, string, error) {
+func UpdateAPI(apiConfig *userconfig.API, models []spec.CuratedModelResource, projectRoot string, projectID string, deployDisallowPrompt bool, awsClient *aws.Client) (*schema.APIResponse, string, error) {
 	telemetry.Event("operator.deploy", apiConfig.TelemetryEvent(types.LocalProviderType))
 
 	var incompatibleVersion string
@@ -83,7 +83,7 @@ func UpdateAPI(apiConfig *userconfig.API, models []spec.CuratedModelResource, co
 		}
 	}
 
-	newAPISpec.LocalProjectDir = files.Dir(configPath)
+	newAPISpec.LocalProjectDir = projectRoot
 
 	if areAPIsEqual(newAPISpec, prevAPISpec) {
 		return toAPIResponse(newAPISpec), fmt.Sprintf("%s is up to date", newAPISpec.Resource.UserString()), nil

cli/local/deploy.go

@@ -30,6 +30,7 @@ import (
 	"github.com/cortexlabs/cortex/pkg/operator/schema"
 	"github.com/cortexlabs/cortex/pkg/types"
 	"github.com/cortexlabs/cortex/pkg/types/spec"
+	"github.com/cortexlabs/cortex/pkg/types/userconfig"
 )
 
 func Deploy(env cliconfig.Environment, configPath string, projectFileList []string, deployDisallowPrompt bool) ([]schema.DeployResult, error) {
@@ -45,11 +46,26 @@ func Deploy(env cliconfig.Environment, configPath string, projectFileList []stri
 		return nil, err
 	}
 
-	projectFiles, err := newProjectFiles(projectFileList, configPath)
+	if !files.IsAbsOrTildePrefixed(configPath) {
+		return nil, errors.ErrorUnexpected(fmt.Sprintf("%s is not an absolute path", configPath))
+	}
+	projectRoot := files.Dir(configPath)
+
+	projectFiles, err := newProjectFiles(projectFileList, projectRoot)
+	if err != nil {
+		return nil, err
+	}
+
+	apiConfigs, err := spec.ExtractAPIConfigs(configBytes, types.LocalProviderType, configFileName, nil, nil)
 	if err != nil {
 		return nil, err
 	}
 
+	return deploy(env, apiConfigs, projectFiles, deployDisallowPrompt)
+}
+
+func deploy(env cliconfig.Environment, apiConfigs []userconfig.API, projectFiles ProjectFiles, deployDisallowPrompt bool) ([]schema.DeployResult, error) {
+	var err error
 	var awsClient *aws.Client
 	var gcpClient *gcp.Client
 
@@ -72,27 +88,23 @@ func Deploy(env cliconfig.Environment, configPath string, projectFileList []stri
 		}
 	}
 
-	apiConfigs, err := spec.ExtractAPIConfigs(configBytes, types.LocalProviderType, configFileName, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-
 	models := []spec.CuratedModelResource{}
 	err = ValidateLocalAPIs(apiConfigs, &models, projectFiles, awsClient, gcpClient)
 	if err != nil {
 		err = errors.Append(err, fmt.Sprintf("\n\napi configuration schema for Realtime API can be found at https://docs.cortex.dev/v/%s/deployments/realtime-api/api-configuration", consts.CortexVersionMinor))
 		return nil, err
 	}
 
-	projectID, err := files.HashFile(projectFileList[0], projectFileList[1:]...)
+	projectRelFilePaths := projectFiles.AllAbsPaths()
+	projectID, err := files.HashFile(projectRelFilePaths[0], projectRelFilePaths[1:]...)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to hash directory", filepath.Dir(configPath))
+		return nil, errors.Wrap(err, "failed to hash directory", projectFiles.projectRoot)
 	}
 
 	results := make([]schema.DeployResult, len(apiConfigs))
 	for i := range apiConfigs {
 		apiConfig := apiConfigs[i]
-		api, msg, err := UpdateAPI(&apiConfig, models, configPath, projectID, deployDisallowPrompt, awsClient)
+		api, msg, err := UpdateAPI(&apiConfig, models, projectFiles.projectRoot, projectID, deployDisallowPrompt, awsClient)
 		results[i].Message = msg
 		if err != nil {
 			results[i].Error = errors.Message(err)

cli/local/patch.go

@@ -0,0 +1,104 @@
+/*
+Copyright 2020 Cortex Labs, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package local
+
+import (
+	"path"
+	"path/filepath"
+
+	"github.com/cortexlabs/cortex/cli/types/cliconfig"
+	"github.com/cortexlabs/cortex/pkg/lib/files"
+	"github.com/cortexlabs/cortex/pkg/operator/schema"
+	"github.com/cortexlabs/cortex/pkg/types"
+	"github.com/cortexlabs/cortex/pkg/types/spec"
+	"github.com/cortexlabs/cortex/pkg/types/userconfig"
+)
+
+func Patch(env cliconfig.Environment, configPath string) ([]schema.DeployResult, error) {
+	configFileName := filepath.Base(configPath)
+
+	configBytes, err := files.ReadFileBytes(configPath)
+	if err != nil {
+		return nil, err
+	}
+
+	apiConfigs, err := spec.ExtractAPIConfigs(configBytes, types.LocalProviderType, configFileName, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	deployResults := []schema.DeployResult{}
+	for i := range apiConfigs {
+		apiConfig := &apiConfigs[i]
+		apiResponse, err := GetAPI(apiConfig.Name)
+		if err != nil {
+			return nil, err
+		}
+
+		localProjectDir := apiResponse[0].Spec.LocalProjectDir
+
+		projectFileList, err := findProjectFiles(localProjectDir)
+		if err != nil {
+			return nil, err
+		}
+
+		projectFiles, err := newProjectFiles(projectFileList, localProjectDir)
+		if err != nil {
+			return nil, err
+		}
+
+		deployResult, err := deploy(env, []userconfig.API{*apiConfig}, projectFiles, true)
+		if err != nil {
+			return nil, err
+		}
+
+		deployResults = append(deployResults, deployResult...)
+	}
+
+	return deployResults, nil
+}
+
+func findProjectFiles(projectRoot string) ([]string, error) {
+	ignoreFns := []files.IgnoreFn{
+		files.IgnoreCortexDebug,
+		files.IgnoreHiddenFiles,
+		files.IgnoreHiddenFolders,
+		files.IgnorePythonGeneratedFiles,
+	}
+
+	cortexIgnorePath := path.Join(projectRoot, ".cortexignore")
+	if files.IsFile(cortexIgnorePath) {
+		cortexIgnore, err := files.GitIgnoreFn(cortexIgnorePath)
+		if err != nil {
+			return nil, err
+		}
+		ignoreFns = append(ignoreFns, cortexIgnore)
+	}
+
+	projectPaths, err := files.ListDirRecursive(projectRoot, false, ignoreFns...)
+	if err != nil {
+		return nil, err
+	}
+
+	// Include .env file containing environment variables
+	dotEnvPath := path.Join(projectRoot, ".env")
+	if files.IsFile(dotEnvPath) {
+		projectPaths = append(projectPaths, dotEnvPath)
+	}
+
+	return projectPaths, nil
+}