Replica pod autoscaling (#196)

Author: deliahu

Makefile

@@ -117,6 +117,7 @@ test-examples:
 ###############
 
 ci-build-images:
+	@./build/build-image.sh images/manager manager
 	@./build/build-image.sh images/spark-base spark-base
 	@./build/build-image.sh images/tf-base tf-base
 	@./build/build-image.sh images/tf-base-gpu tf-base-gpu
@@ -135,9 +136,10 @@ ci-build-images:
 	@./build/build-image.sh images/argo-executor argo-executor
 	@./build/build-image.sh images/python-packager python-packager
 	@./build/build-image.sh images/cluster-autoscaler cluster-autoscaler
-	@./build/build-image.sh images/manager manager
+	@./build/build-image.sh images/metrics-server metrics-server
 
 ci-push-images:
+	@./build/push-image.sh manager
 	@./build/push-image.sh spark
 	@./build/push-image.sh spark-operator
 	@./build/push-image.sh tf-train
@@ -153,7 +155,8 @@ ci-push-images:
 	@./build/push-image.sh argo-executor
 	@./build/push-image.sh python-packager
 	@./build/push-image.sh cluster-autoscaler
-	@./build/push-image.sh manager
+	@./build/push-image.sh metrics-server
+
 
 ci-build-cli:
 	@./build/cli.sh

README.md

@@ -26,8 +26,8 @@ Cortex is actively maintained by Cortex Labs. We're a venture-backed team of inf
   model: s3://my-bucket/my-model.zip
   request_handler: handler.py
   compute:
-    replicas: 4
-    gpu: 2
+    min_replicas: 5
+    max_replicas: 20
 ```
 
 **Customize** request handling (optional):

cli/cmd/get.go

@@ -443,16 +443,16 @@ func describeAPI(name string, resourcesRes *schema.GetResourcesResponse) (string
 	}
 
 	out := titleStr("Summary")
-	out += "Status:            " + groupStatus.Message() + "\n"
+	out += "Status:               " + groupStatus.Message() + "\n"
 	if ctxAPIStatus != nil {
-		out += fmt.Sprintf("Updated replicas:  %d/%d ready\n", ctxAPIStatus.ReadyUpdated, ctxAPIStatus.RequestedReplicas)
+		out += fmt.Sprintf("Up-to-date replicas:  %d ready\n", ctxAPIStatus.ReadyUpdated)
 	}
 	if staleReplicas != 0 {
-		out += fmt.Sprintf("Stale replicas:    %d ready\n", staleReplicas)
+		out += fmt.Sprintf("Stale replicas:       %d ready\n", staleReplicas)
 	}
-	out += "Created at:        " + libtime.LocalTimestamp(groupStatus.Start) + "\n"
+	out += "Created at:           " + libtime.LocalTimestamp(groupStatus.Start) + "\n"
 	if groupStatus.ActiveStatus != nil && groupStatus.ActiveStatus.Start != nil {
-		out += "Refreshed at:      " + libtime.LocalTimestamp(groupStatus.ActiveStatus.Start) + "\n"
+		out += "Refreshed at:         " + libtime.LocalTimestamp(groupStatus.ActiveStatus.Start) + "\n"
 	}
 
 	out += titleStr("Endpoint")

cortex.sh

@@ -131,6 +131,7 @@ export CORTEX_IMAGE_PYTHON_PACKAGER="${CORTEX_IMAGE_PYTHON_PACKAGER:-cortexlabs/
 export CORTEX_IMAGE_TF_SERVE_GPU="${CORTEX_IMAGE_TF_SERVE_GPU:-cortexlabs/tf-serve-gpu:$CORTEX_VERSION_STABLE}"
 export CORTEX_IMAGE_TF_TRAIN_GPU="${CORTEX_IMAGE_TF_TRAIN_GPU:-cortexlabs/tf-train-gpu:$CORTEX_VERSION_STABLE}"
 export CORTEX_IMAGE_CLUSTER_AUTOSCALER="${CORTEX_IMAGE_CLUSTER_AUTOSCALER:-cortexlabs/cluster-autoscaler:$CORTEX_VERSION_STABLE}"
+export CORTEX_IMAGE_METRICS_SERVER="${CORTEX_IMAGE_METRICS_SERVER:-cortexlabs/metrics-server:$CORTEX_VERSION_STABLE}"
 
 export CORTEX_ENABLE_TELEMETRY="${CORTEX_ENABLE_TELEMETRY:-""}"
 
@@ -181,6 +182,7 @@ function install_cortex() {
     -e CORTEX_IMAGE_TF_SERVE_GPU=$CORTEX_IMAGE_TF_SERVE_GPU \
     -e CORTEX_IMAGE_TF_TRAIN_GPU=$CORTEX_IMAGE_TF_TRAIN_GPU \
     -e CORTEX_IMAGE_CLUSTER_AUTOSCALER=$CORTEX_IMAGE_CLUSTER_AUTOSCALER \
+    -e CORTEX_IMAGE_METRICS_SERVER=$CORTEX_IMAGE_METRICS_SERVER \
     -e CORTEX_ENABLE_TELEMETRY=$CORTEX_ENABLE_TELEMETRY \
     $CORTEX_IMAGE_MANAGER
 }

dev/load.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Copyright 2019 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+SLEEP="0.01"
+
+URL="https://ac47cdf2e99cb11e9ade10693263c378-1155928797.us-west-2.elb.amazonaws.com/iris/iris-type"
+DATA='{ "samples": [ { "sepal_length": 5.2, "sepal_width": 3.6, "petal_length": 1.4, "petal_width": 0.3 } ] }'
+
+# URL="https://ac47cdf2e99cb11e9ade10693263c378-1155928797.us-west-2.elb.amazonaws.com/insurance/cost"
+# DATA='{ "samples": [ { "age": 22, "bmi": 25, "children": 0, "region": "northeast", "sex": "female", "smoker": "no" } ] }'
+
+trap ctrl_c INT
+function ctrl_c() {
+  echo ""
+  exit 0
+}
+
+function make_request() {
+  curl --silent --show-error -k -X POST -H "Content-Type: application/json" -d "${DATA}" "${URL}"
+}
+
+resp=$(make_request)
+echo -n "."
+
+while eval "sleep ${SLEEP}"; do
+  resp=$(make_request)
+  echo -n "."
+done

dev/registry.sh

@@ -35,6 +35,7 @@ function ecr_login() {
 }
 
 function create_registry() {
+  aws ecr create-repository --repository-name=cortexlabs/manager --region=$REGISTRY_REGION || true
   aws ecr create-repository --repository-name=cortexlabs/argo-controller --region=$REGISTRY_REGION || true
   aws ecr create-repository --repository-name=cortexlabs/argo-executor --region=$REGISTRY_REGION || true
   aws ecr create-repository --repository-name=cortexlabs/fluentd --region=$REGISTRY_REGION || true
@@ -50,7 +51,7 @@ function create_registry() {
   aws ecr create-repository --repository-name=cortexlabs/tf-train-gpu --region=$REGISTRY_REGION || true
   aws ecr create-repository --repository-name=cortexlabs/tf-serve-gpu --region=$REGISTRY_REGION || true
   aws ecr create-repository --repository-name=cortexlabs/cluster-autoscaler --region=$REGISTRY_REGION || true
-  aws ecr create-repository --repository-name=cortexlabs/manager --region=$REGISTRY_REGION || true
+  aws ecr create-repository --repository-name=cortexlabs/metrics-server --region=$REGISTRY_REGION || true
 }
 
 ### HELPERS ###
@@ -116,6 +117,8 @@ if [ "$cmd" = "create" ]; then
 
 elif [ "$cmd" = "update" ]; then
   if [ "$env" != "dev" ]; then
+    build_and_push $ROOT/images/manager manager latest
+
     cache_builder $ROOT/images/spark-base spark-base
     build_base $ROOT/images/spark-base spark-base
     build_base $ROOT/images/tf-base tf-base
@@ -136,7 +139,7 @@ elif [ "$cmd" = "update" ]; then
     build_and_push $ROOT/images/tf-serve-gpu tf-serve-gpu latest
     build_and_push $ROOT/images/python-packager python-packager latest
     build_and_push $ROOT/images/cluster-autoscaler cluster-autoscaler latest
-    build_and_push $ROOT/images/manager manager latest
+    build_and_push $ROOT/images/metrics-server metrics-server latest
   fi
 
   build_and_push $ROOT/images/spark spark latest

docs/apis/apis.md

@@ -9,9 +9,12 @@ Serve models at scale and use them to build smarter applications.
   name: <string>  # API name (required)
   model: <string>  # path to a zipped model dir (e.g. s3://my-bucket/model.zip)
   compute:
-    replicas: <int>  # number of replicas to launch (default: 1)
-    cpu: <string>  # CPU request per replica (default: Null)
-    gpu: <string>  # gpu request per replica (default: Null)
+    min_replicas: <int>  # minimum number of replicas (default: 1)
+    max_replicas: <int>  # maximum number of replicas (default: 100)
+    init_replicas: <int>  # initial number of replicas (default: <min_replicas>)
+    target_cpu_utilization: <int>  # CPU utilization threshold (as a percentage) to trigger scaling (default: 80)
+    cpu: <string>  # CPU request per replica (default: 200m)
+    gpu: <string>  # gpu request per replica (default: 0)
     mem: <string>  # memory request per replica (default: Null)
 ```
 
@@ -24,8 +27,9 @@ See [packaging models](packaging-models.md) for how to create the zipped model.
   name: my-api
   model: s3://my-bucket/my-model.zip
   compute:
-    replicas: 3
-    gpu: 2
+    min_replicas: 5
+    max_replicas: 20
+    cpu: "1"
 ```
 
 ## Integration

docs/apis/tutorial.md

@@ -33,8 +33,6 @@ Add to `cortex.yaml`:
 - kind: api
   name: iris-type
   model: s3://cortex-examples/iris-tensorflow.zip
-  compute:
-    replicas: 3
 ```
 
 ### Deploy the API

docs/cluster/config.md

@@ -52,6 +52,7 @@ export CORTEX_IMAGE_TF_TRAIN_GPU="cortexlabs/tf-train-gpu:master"
 export CORTEX_IMAGE_TF_SERVE_GPU="cortexlabs/tf-serve-gpu:master"
 export CORTEX_IMAGE_PYTHON_PACKAGER="cortexlabs/python-packager:master"
 export CORTEX_IMAGE_CLUSTER_AUTOSCALER="cortexlabs/cluster-autoscaler:master"
+export CORTEX_IMAGE_METRICS_SERVER="cortexlabs/metrics-server:master"
 
 # Flag to enable collecting error reports and usage stats. If flag is not set to either "true" or "false", you will be prompted.
 export CORTEX_ENABLE_TELEMETRY=""

docs/cluster/development.md

@@ -86,6 +86,7 @@ export CORTEX_IMAGE_TF_TRAIN_GPU="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/corte
 export CORTEX_IMAGE_TF_TRANSFORM="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/tf-transform:latest"
 export CORTEX_IMAGE_PYTHON_PACKAGER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/python-packager:latest"
 export CORTEX_IMAGE_CLUSTER_AUTOSCALER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/cluster-autoscaler:latest"
+export CORTEX_IMAGE_METRICS_SERVER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/metrics-server:latest"
 
 export AWS_ACCESS_KEY_ID="XXXXXX"
 export AWS_SECRET_ACCESS_KEY="XXXXXX"