Remove region from external data

Author: deliahu

cli/cmd/get.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cortexlabs/yaml"
 	"github.com/spf13/cobra"
 
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
@@ -412,8 +413,8 @@ func describeAPI(name string, resourcesRes *schema.GetResourcesResponse) (string
 	out += "Method:   POST\n"
 	out += `Header:   "Content-Type: application/json"` + "\n"
 
-	if api.Model != nil {
-		model := ctx.Models[api.ModelName]
+	if modelName, ok := yaml.ExtractAtSymbolText(api.Model); ok {
+		model := ctx.Models[modelName]
 		resIDs := strset.New()
 		combinedInput := []interface{}{model.Input, model.TrainingInput}
 		for _, res := range ctx.ExtractCortexResources(combinedInput, resource.ConstantType, resource.RawColumnType, resource.AggregateType, resource.TransformedColumnType) {

docs/apis/apis.md

@@ -7,9 +7,7 @@ Serve models at scale and use them to build smarter applications.
 ```yaml
 - kind: api
   name: <string>  # API name (required)
-  external_model:
-    path: <string>  # path to a zipped model dir (e.g. s3://my-bucket/model.zip)
-    region: <string>  # S3 region (default: us-west-2)
+  model: <string>  # path to a zipped model dir (e.g. s3://my-bucket/model.zip)
   compute:
     replicas: <int>  # number of replicas to launch (default: 1)
     cpu: <string>  # CPU request per replica (default: Null)
@@ -24,9 +22,7 @@ See [packaging models](packaging-models.md) for how to create the zipped model.
 ```yaml
 - kind: api
   name: my-api
-  external_model:
-    path: s3://my-bucket/my-model.zip
-    region: us-west-2
+  model: s3://my-bucket/my-model.zip
   compute:
     replicas: 3
     gpu: 2

docs/apis/packaging-models.md

@@ -17,12 +17,10 @@ Upload the zipped file to Amazon S3, e.g.
 $ aws s3 cp model.zip s3://my-bucket/model.zip
 ```
 
-Specify `external_model` in an API, e.g.
+Specify `model` in an API, e.g.
 
 ```yaml
 - kind: api
   name: my-api
-  external_model:
-    path: s3://my-bucket/model.zip
-    region: us-west-2
+  model: s3://my-bucket/model.zip
 ```

docs/apis/tutorial.md

@@ -32,9 +32,7 @@ Add to `cortex.yaml`:
 ```yaml
 - kind: api
   name: iris-type
-  external_model:
-    path: s3://cortex-examples/iris-tensorflow.zip
-    region: us-west-2
+  model: s3://cortex-examples/iris-tensorflow.zip
   compute:
     replicas: 3
 ```

docs/pipelines/apis.md

@@ -7,18 +7,15 @@ Serve models at scale and use them to build smarter applications.
 ```yaml
 - kind: api
   name: <string>  # API name (required)
-  model: <string>  # reference to a model (e.g. @dnn) (this or external_model must be specified)
-  external_model:  # (this or model must be specified)
-    path: <string>  # path to a zipped model dir (e.g. s3://my-bucket/model.zip)
-    region: <string>  # S3 region (default: us-west-2)
+  model: <string>  # reference to a model (e.g. @dnn) or path to a zipped model dir (e.g. s3://my-bucket/model.zip)
   compute:
     replicas: <int>  # number of replicas to launch (default: 1)
     cpu: <string>  # CPU request per replica (default: Null)
     gpu: <string>  # gpu request per replica (default: Null)
     mem: <string>  # memory request per replica (default: Null)
 ```
 
-See [packaging models](packaging-models.md) for how to create the zipped model (for use in `external_model`).
+See [packaging models](packaging-models.md) for how to create a zipped model (only if using an externally-built models).
 
 ## Example
 

docs/pipelines/constants.md

@@ -8,10 +8,8 @@ Constants represent literal values which can be used in other Cortex resources.
 - kind: constant
   name: <string>  # constant name (required)
   type: <output_type>  # the type of the constant (optional, will be inferred from value if not specified)
-  value: <output_value>  # a literal value (this or external_model must be specified)
-  external_model:  # (this or value must be specified)
-    path: <string>  # path to a JSON object (e.g. s3://my-bucket/constant.json)
-    region: <string>  # S3 region (default: us-west-2)
+  value: <output_value>  # a literal value (this or path must be specified)
+  path: <string>  # path to a JSON object (e.g. s3://my-bucket/constant.json) (this or value must be specified)
 
 ```
 

docs/pipelines/environments.md

@@ -26,7 +26,6 @@ Transfer data at scale from data warehouses like S3 into the Cortex environment.
 data:
   type: csv  # file type (required)
   path: s3a://<bucket_name>/<file_name>  # S3 is currently supported (required)
-  region: us-west-2  # S3 region (default: us-west-2)
   drop_null: <bool>  # drop any rows that contain at least 1 null value (default: false)
   csv_config: <csv_config>  # optional configuration that can be provided
   schema:
@@ -65,7 +64,6 @@ csv_config:
 data:
   type: parquet  # file type (required)
   path: s3a://<bucket_name>/<file_name>  # S3 is currently supported (required)
-  region: us-west-2  # S3 region (default: us-west-2)
   drop_null: <bool>  # drop any rows that contain at least 1 null value (default: false)
   schema:
     - parquet_column_name: <string>  # name of the column in the parquet file (required)

docs/pipelines/packaging-models.md

@@ -17,12 +17,10 @@ Upload the zipped file to Amazon S3, e.g.
 $ aws s3 cp model.zip s3://my-bucket/model.zip
 ```
 
-Specify `external_model` in an API, e.g.
+Specify `model` in an API, e.g.
 
 ```yaml
 - kind: api
   name: my-api
-  external_model:
-    path: s3://my-bucket/model.zip
-    region: us-west-2
+  model: s3://my-bucket/model.zip
 ```

examples/iris/cortex.yaml

@@ -3,8 +3,6 @@
 
 - kind: api
   name: iris-type
-  external_model:
-    path: s3://cortex-examples/iris-tensorflow.zip
-    region: us-west-2
+  model: s3://cortex-examples/iris-tensorflow.zip
   compute:
     replicas: 3

go.mod

@@ -5,7 +5,7 @@
 //   replace these lines in go.mod:
 //     github.com/GoogleCloudPlatform/spark-on-k8s-operator v1alpha1-0.5-2.4.0
 //     github.com/argoproj/argo v2.3.0
-//     github.com/cortexlabs/yaml v2.2.3
+//     github.com/cortexlabs/yaml v2.2.4
 //     k8s.io/client-go v10.0.0
 //     k8s.io/api 89a74a8d264df0e993299876a8cde88379b940ee
 //     k8s.io/apimachinery 2b1284ed4c93a43499e781493253e2ac5959c4fd
@@ -20,8 +20,8 @@ go 1.12
 require (
 	github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.0.0-20181208011959-62db1d66dafa
 	github.com/argoproj/argo v2.3.0+incompatible
-	github.com/aws/aws-sdk-go v1.20.7
-	github.com/cortexlabs/yaml v0.0.0-20190624201412-7f31702857b6
+	github.com/aws/aws-sdk-go v1.20.8
+	github.com/cortexlabs/yaml v0.0.0-20190626164117-202ab3a3d475
 	github.com/davecgh/go-spew v1.1.1
 	github.com/emicklei/go-restful v2.9.6+incompatible // indirect
 	github.com/go-openapi/spec v0.19.2 // indirect