Update dev instructions

Author: deliahu

dev/aws.sh

@@ -20,7 +20,6 @@ set -euo pipefail
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 
 source $ROOT/dev/config/build.sh
-source $ROOT/dev/config/k8s.sh
 source $ROOT/dev/config/cortex.sh
 source $ROOT/dev/util.sh
 

docs/cluster/development.md

@@ -22,13 +22,12 @@ Create the AWS Elastic Container Registry:
 make registry-create
 ```
 
-Note the registry URL, this will be needed shortly.
+Take note of the registry URL, this will be needed shortly.
 
 Create the S3 buckets:
 
 ```bash
 aws s3 mb s3://cortex-cluster-<your_name>
-aws s3 mb s3://cortex-kops-<your_name> # (if you'll be using KOPS)
 aws s3 mb s3://cortex-cli-<your_name> # (if you'll be uploading your compiled CLI)
 ```
 
@@ -40,37 +39,23 @@ Make the config folder:
 mkdir dev/config
 ```
 
-Create `dev/config/k8s.sh`. Paste the following config, replace `K8S_KOPS_BUCKET` with your bucket name (if using KOPS), and update any other variables as desired:
-
-```bash
-# EKS and KOPS
-
-export K8S_NAME="cortex"
-export K8S_REGION="us-west-2"
-export K8S_NODE_INSTANCE_TYPE="t3.medium"
-export K8S_NODES_MAX_COUNT="2"
-export K8S_NODES_MIN_COUNT="2"
-export K8S_GPU_NODES_MIN_COUNT="0"
-export K8S_GPU_NODES_MAX_COUNT="0"
-
-# KOPS only
-
-export K8S_KOPS_BUCKET="cortex-kops-<your_name>"
-export K8S_ZONE="us-west-2a"
-export K8S_MASTER_INSTANCE_TYPE="t3.micro"
-export K8S_MASTER_VOLUME_SIZE="32"
-export K8S_NODE_VOLUME_SIZE="32"
-```
-
 Create `dev/config/cortex.sh`. Paste the following config, and update `CORTEX_BUCKET`, `CORTEX_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and all registry URLs accordingly:
 
 ```bash
+export AWS_ACCESS_KEY_ID="XXXXXX"
+export AWS_SECRET_ACCESS_KEY="XXXXXX"
+
 export CORTEX_LOG_GROUP="cortex"
 export CORTEX_BUCKET="cortex-cluster-<your_name>"
 export CORTEX_REGION="us-west-2"
+
+export CORTEX_CLUSTER="cortex"
+export CORTEX_NODE_TYPE="t3.small"
+export CORTEX_NODES_MIN="2"
+export CORTEX_NODES_MAX="5"
 export CORTEX_NAMESPACE="cortex"
-export CORTEX_ENABLE_TELEMETRY="false"
 
+export CORTEX_IMAGE_MANAGER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/manager:latest"
 export CORTEX_IMAGE_ARGO_CONTROLLER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/argo-controller:latest"
 export CORTEX_IMAGE_ARGO_EXECUTOR="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/argo-executor:latest"
 export CORTEX_IMAGE_FLUENTD="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/fluentd:latest"
@@ -83,14 +68,13 @@ export CORTEX_IMAGE_TF_SERVE="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlab
 export CORTEX_IMAGE_TF_SERVE_GPU="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/tf-serve-gpu:latest"
 export CORTEX_IMAGE_TF_TRAIN="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/tf-train:latest"
 export CORTEX_IMAGE_TF_TRAIN_GPU="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/tf-train-gpu:latest"
-export CORTEX_IMAGE_TF_TRANSFORM="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/tf-transform:latest"
+export CORTEX_IMAGE_TF_API="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/tf-api:latest"
 export CORTEX_IMAGE_PYTHON_PACKAGER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/python-packager:latest"
 export CORTEX_IMAGE_CLUSTER_AUTOSCALER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/cluster-autoscaler:latest"
 export CORTEX_IMAGE_NVIDIA="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/nvidia:latest"
 export CORTEX_IMAGE_METRICS_SERVER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/metrics-server:latest"
 
-export AWS_ACCESS_KEY_ID="XXXXXX"
-export AWS_SECRET_ACCESS_KEY="XXXXXX"
+export CORTEX_ENABLE_TELEMETRY="false"
 ```
 
 Create `dev/config/build.sh`. Paste the following config, and update `CLI_BUCKET_NAME`, `CLI_BUCKET_REGION`, `REGISTRY_URL`, and `REGISTRY_REGION` accordingly:
@@ -106,73 +90,61 @@ export CLI_BUCKET_NAME="cortex-cli-<your_name>"
 export CLI_BUCKET_REGION="us-west-2"
 ```
 
-### Kubernetes
+### Building
 
-Start Kubernetes cluster and install Cortex on it:
+Build and push all Cortex images:
 
 ```bash
-make eks-up
-# or
-make kops-up
+make registry-all
 ```
 
-If you're using GPUs on EKS, run this after your GPU nodes join the cluster:
+Build and configure the Cortex CLI:
 
 ```bash
-kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v1.11/nvidia-device-plugin.yml
-# check for GPUs:
-kubectl get nodes "-o=custom-columns=NAME:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu"
+make cli  # The binary will be placed in path/to/cortex/bin/cortex
+path/to/cortex/bin/cortex configure
 ```
 
-### Building
+### Cortex Cluster
 
-Build and push all Cortex images (this will take a while):
+Start Cortex:
 
 ```bash
-make registry-all
+make cortex-up
 ```
 
-Build and configure the Cortex CLI:
+Tear down the Cortex cluster:
 
 ```bash
-make cli  # The binary will be placed in path/to/cortex/bin/cortex
-path/to/cortex/bin/cortex configure
+make cortex-down
 ```
 
-### Deployment
-
-Deploy an example:
+### Deployment an Example
 
 ```bash
 cd examples/iris
 path/to/cortex/bin/cortex deploy
 ```
 
-Tear down the cluster:
-
-```bash
-make eks-down
-# or
-make kops-down
-```
-
 ## Off-cluster Operator
 
 If you're making changes in the operator and want faster iterations, you can run an off-cluster operator.
 
-1. `make ostop` to stop the in-cluster operator
+1. `make operator-stop` to stop the in-cluster operator
 1. `make devstart` to run the off-cluster operator (which rebuilds the CLI and restarts the Operator when files change)
 1. `path/to/cortex/bin/cortex configure` (on a separate terminal) to configure your cortex CLI to use the off-cluster operator. When prompted for operator URL, use `http://localhost:8888`
 
+Note: `make cortex-up-dev` will start Cortex without installing the operator.
+
 If you want to switch back to the in-cluster operator:
 
 1. `<ctrl+C>` to stop your off-cluster operator
-1. `oinstall` to install the operator in your cluster
-1. `path/to/cortex/bin/cortex configure` to configure your cortex CLI to use the in-cluster operator. When prompted for operator URL, use the URL shown when running `oinstall`
+1. `make operator-start` to install the operator in your cluster
+1. `path/to/cortex/bin/cortex configure` to configure your cortex CLI to use the in-cluster operator. When prompted for operator URL, use the URL shown when running `make cortex-info`
 
 ## Dev Workflow
 
-1. Install
+1. `make cortex-up-dev`
 1. `make devstart`
 1. Make changes
 1. `make registry-dev`