Switch to Quay docker registry (#1578)

deliahu · web-flow · commit 90ec64b49062 · 2020-11-20T09:31:24.000-08:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -45,6 +45,9 @@ jobs:
   build-and-deploy:
     docker:
       - image: circleci/python:3.6
+        auth:
+          username: $QUAY_USERNAME
+          password: $QUAY_PASSWORD
     environment:
       NUM_BUILD_PROCS: "2"
     steps:
diff --git a/build/build-image.sh b/build/build-image.sh
@@ -20,6 +20,7 @@ set -euo pipefail
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 
 CORTEX_VERSION=master
+REGISTRY_URL=quay.io
 
 image=$1
 dir="${ROOT}/images/${image/-slim}"
@@ -38,8 +39,8 @@ if [ "${image}" == "python-predictor-gpu-slim" ]; then
   cudnn=("7" "7" "8" "7" "8" "8" "8")
   for i in ${!cudnn[@]}; do
     build_args="${build_args} --build-arg CUDA_VERSION=${cuda[$i]} --build-arg CUDNN=${cudnn[$i]}"
-    docker build "$ROOT" -f $dir/Dockerfile $build_args -t cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]}
+    docker build "$ROOT" -f $dir/Dockerfile $build_args -t $REGISTRY_URL/cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]}
   done
 else
-  docker build "$ROOT" -f $dir/Dockerfile $build_args -t cortexlabs/${image}:${CORTEX_VERSION}
+  docker build "$ROOT" -f $dir/Dockerfile $build_args -t $REGISTRY_URL/cortexlabs/${image}:${CORTEX_VERSION}
 fi
diff --git a/build/push-image.sh b/build/push-image.sh
@@ -18,6 +18,7 @@
 set -euo pipefail
 
 CORTEX_VERSION=master
+REGISTRY_URL=quay.io
 
 image=$1
 
@@ -27,8 +28,8 @@ if [ "$image" == "python-predictor-gpu-slim" ]; then
   cuda=("10.0" "10.1" "10.1" "10.2" "10.2" "11.0" "11.1")
   cudnn=("7" "7" "8" "7" "8" "8" "8")
   for i in ${!cudnn[@]}; do
-    docker push cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]}
+    docker push $REGISTRY_URL/cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]}
   done
 else
-  docker push cortexlabs/${image}:${CORTEX_VERSION}
+  docker push $REGISTRY_URL/cortexlabs/${image}:${CORTEX_VERSION}
 fi
diff --git a/docs/cluster-management/config.md b/docs/cluster-management/config.md
@@ -84,17 +84,17 @@ The docker images used by the Cortex cluster can also be overridden, although th
 <!-- CORTEX_VERSION_BRANCH_STABLE -->
 ```yaml
 # docker image paths
-image_operator: cortexlabs/operator:master
-image_manager: cortexlabs/manager:master
-image_downloader: cortexlabs/downloader:master
-image_request_monitor: cortexlabs/request-monitor:master
-image_cluster_autoscaler: cortexlabs/cluster-autoscaler:master
-image_metrics_server: cortexlabs/metrics-server:master
-image_inferentia: cortexlabs/inferentia:master
-image_neuron_rtd: cortexlabs/neuron-rtd:master
-image_nvidia: cortexlabs/nvidia:master
-image_fluentd: cortexlabs/fluentd:master
-image_statsd: cortexlabs/statsd:master
-image_istio_proxy: cortexlabs/istio-proxy:master
-image_istio_pilot: cortexlabs/istio-pilot:master
+image_operator: quay.io/cortexlabs/operator:master
+image_manager: quay.io/cortexlabs/manager:master
+image_downloader: quay.io/cortexlabs/downloader:master
+image_request_monitor: quay.io/cortexlabs/request-monitor:master
+image_cluster_autoscaler: quay.io/cortexlabs/cluster-autoscaler:master
+image_metrics_server: quay.io/cortexlabs/metrics-server:master
+image_inferentia: quay.io/cortexlabs/inferentia:master
+image_neuron_rtd: quay.io/cortexlabs/neuron-rtd:master
+image_nvidia: quay.io/cortexlabs/nvidia:master
+image_fluentd: quay.io/cortexlabs/fluentd:master
+image_statsd: quay.io/cortexlabs/statsd:master
+image_istio_proxy: quay.io/cortexlabs/istio-proxy:master
+image_istio_pilot: quay.io/cortexlabs/istio-pilot:master
 ```
diff --git a/docs/deployments/batch-api/api-configuration.md b/docs/deployments/batch-api/api-configuration.md
@@ -16,7 +16,7 @@ Reference the section below which corresponds to your Predictor type: [Python](#
     path: <string>  # path to a python file with a PythonPredictor class definition, relative to the Cortex root (required)
     config: <string: value>  # arbitrary dictionary passed to the constructor of the Predictor (can be overridden by config passed in job submission) (optional)
     python_path: <string>  # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml)
-    image: <string> # docker image to use for the Predictor (default: cortexlabs/python-predictor-cpu or cortexlabs/python-predictor-gpu based on compute)
+    image: <string> # docker image to use for the Predictor (default: quay.io/cortexlabs/python-predictor-cpu or quay.io/cortexlabs/python-predictor-gpu based on compute)
     env: <string: string>  # dictionary of environment variables
   networking:
     endpoint: <string>  # the endpoint for the API (default: <api_name>)
@@ -50,8 +50,8 @@ See additional documentation for [compute](../compute.md), [networking](../netwo
       batch_interval: <duration>  # the maximum amount of time to spend waiting for additional requests before running inference on the batch of requests
     config: <string: value>  # arbitrary dictionary passed to the constructor of the Predictor (can be overridden by config passed in job submission) (optional)
     python_path: <string>  # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml)
-    image: <string> # docker image to use for the Predictor (default: cortexlabs/tensorflow-predictor)
-    tensorflow_serving_image: <string> # docker image to use for the TensorFlow Serving container (default: cortexlabs/tensorflow-serving-gpu or cortexlabs/tensorflow-serving-cpu based on compute)
+    image: <string> # docker image to use for the Predictor (default: quay.io/cortexlabs/tensorflow-predictor)
+    tensorflow_serving_image: <string> # docker image to use for the TensorFlow Serving container (default: quay.io/cortexlabs/tensorflow-serving-gpu or quay.io/cortexlabs/tensorflow-serving-cpu based on compute)
     env: <string: string>  # dictionary of environment variables
   networking:
     endpoint: <string>  # the endpoint for the API (default: <api_name>)
@@ -81,7 +81,7 @@ See additional documentation for [compute](../compute.md), [networking](../netwo
       ...
     config: <string: value>  # arbitrary dictionary passed to the constructor of the Predictor (can be overridden by config passed in job submission) (optional)
     python_path: <string>  # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml)
-    image: <string> # docker image to use for the Predictor (default: cortexlabs/onnx-predictor-gpu or cortexlabs/onnx-predictor-cpu based on compute)
+    image: <string> # docker image to use for the Predictor (default: quay.io/cortexlabs/onnx-predictor-gpu or quay.io/cortexlabs/onnx-predictor-cpu based on compute)
     env: <string: string>  # dictionary of environment variables
   networking:
     endpoint: <string>  # the endpoint for the API (default: <api_name>)
diff --git a/docs/deployments/realtime-api/api-configuration.md b/docs/deployments/realtime-api/api-configuration.md
@@ -27,7 +27,7 @@ Reference the section below which corresponds to your Predictor type: [Python](#
     threads_per_process: <int>  # the number of threads per process (default: 1)
     config: <string: value>  # arbitrary dictionary passed to the constructor of the Predictor (optional)
     python_path: <string>  # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml)
-    image: <string>  # docker image to use for the Predictor (default: cortexlabs/python-predictor-cpu or cortexlabs/python-predictor-gpu based on compute)
+    image: <string>  # docker image to use for the Predictor (default: quay.io/cortexlabs/python-predictor-cpu or quay.io/cortexlabs/python-predictor-gpu based on compute)
     env: <string: string>  # dictionary of environment variables
   networking:
     endpoint: <string>  # the endpoint for the API (aws only) (default: <api_name>)
@@ -88,8 +88,8 @@ See additional documentation for [models](models.md), [parallelism](parallelism.
     threads_per_process: <int>  # the number of threads per process (default: 1)
     config: <string: value>  # arbitrary dictionary passed to the constructor of the Predictor (optional)
     python_path: <string>  # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml)
-    image: <string>  # docker image to use for the Predictor (default: cortexlabs/tensorflow-predictor)
-    tensorflow_serving_image: <string>  # docker image to use for the TensorFlow Serving container (default: cortexlabs/tensorflow-serving-gpu or cortexlabs/tensorflow-serving-cpu based on compute)
+    image: <string>  # docker image to use for the Predictor (default: quay.io/cortexlabs/tensorflow-predictor)
+    tensorflow_serving_image: <string>  # docker image to use for the TensorFlow Serving container (default: quay.io/cortexlabs/tensorflow-serving-gpu or quay.io/cortexlabs/tensorflow-serving-cpu based on compute)
     env: <string: string>  # dictionary of environment variables
   networking:
     endpoint: <string>  # the endpoint for the API (aws only) (default: <api_name>)
@@ -145,7 +145,7 @@ See additional documentation for [models](models.md), [parallelism](parallelism.
     threads_per_process: <int>  # the number of threads per process (default: 1)
     config: <string: value>  # arbitrary dictionary passed to the constructor of the Predictor (optional)
     python_path: <string>  # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml)
-    image: <string>  # docker image to use for the Predictor (default: cortexlabs/onnx-predictor-gpu or cortexlabs/onnx-predictor-cpu based on compute)
+    image: <string>  # docker image to use for the Predictor (default: quay.io/cortexlabs/onnx-predictor-gpu or quay.io/cortexlabs/onnx-predictor-cpu based on compute)
     env: <string: string>  # dictionary of environment variables
   networking:
     endpoint: <string>  # the endpoint for the API (aws only) (default: <api_name>)
diff --git a/docs/deployments/system-packages.md b/docs/deployments/system-packages.md
@@ -49,19 +49,19 @@ mkdir my-api && cd my-api && touch Dockerfile
 Cortex's base Docker images are listed below. Depending on the Cortex Predictor and compute type specified in your API configuration, choose one of these images to use as the base for your Docker image:
 
 <!-- CORTEX_VERSION_BRANCH_STABLE x12 -->
-* Python Predictor (CPU): `cortexlabs/python-predictor-cpu-slim:master`
+* Python Predictor (CPU): `quay.io/cortexlabs/python-predictor-cpu-slim:master`
 * Python Predictor (GPU): choose one of the following:
-  * `cortexlabs/python-predictor-gpu-slim:master-cuda10.0-cudnn7`
-  * `cortexlabs/python-predictor-gpu-slim:master-cuda10.1-cudnn7`
-  * `cortexlabs/python-predictor-gpu-slim:master-cuda10.1-cudnn8`
-  * `cortexlabs/python-predictor-gpu-slim:master-cuda10.2-cudnn7`
-  * `cortexlabs/python-predictor-gpu-slim:master-cuda10.2-cudnn8`
-  * `cortexlabs/python-predictor-gpu-slim:master-cuda11.0-cudnn8`
-  * `cortexlabs/python-predictor-gpu-slim:master-cuda11.1-cudnn8`
-* Python Predictor (Inferentia): `cortexlabs/python-predictor-inf-slim:master`
-* TensorFlow Predictor (CPU, GPU, Inferentia): `cortexlabs/tensorflow-predictor-slim:master`
-* ONNX Predictor (CPU): `cortexlabs/onnx-predictor-cpu-slim:master`
-* ONNX Predictor (GPU): `cortexlabs/onnx-predictor-gpu-slim:master`
+  * `quay.io/cortexlabs/python-predictor-gpu-slim:master-cuda10.0-cudnn7`
+  * `quay.io/cortexlabs/python-predictor-gpu-slim:master-cuda10.1-cudnn7`
+  * `quay.io/cortexlabs/python-predictor-gpu-slim:master-cuda10.1-cudnn8`
+  * `quay.io/cortexlabs/python-predictor-gpu-slim:master-cuda10.2-cudnn7`
+  * `quay.io/cortexlabs/python-predictor-gpu-slim:master-cuda10.2-cudnn8`
+  * `quay.io/cortexlabs/python-predictor-gpu-slim:master-cuda11.0-cudnn8`
+  * `quay.io/cortexlabs/python-predictor-gpu-slim:master-cuda11.1-cudnn8`
+* Python Predictor (Inferentia): `quay.io/cortexlabs/python-predictor-inf-slim:master`
+* TensorFlow Predictor (CPU, GPU, Inferentia): `quay.io/cortexlabs/tensorflow-predictor-slim:master`
+* ONNX Predictor (CPU): `quay.io/cortexlabs/onnx-predictor-cpu-slim:master`
+* ONNX Predictor (GPU): `quay.io/cortexlabs/onnx-predictor-gpu-slim:master`
 
 Note: the images listed above use the `-slim` suffix; Cortex's default API images are not `-slim`, since they have additional dependencies installed to cover common use cases. If you are building your own Docker image, starting with a `-slim` Predictor image will result in a smaller image size.
 
@@ -71,7 +71,7 @@ The sample Dockerfile below inherits from Cortex's Python CPU serving image, and
 ```dockerfile
 # Dockerfile
 
-FROM cortexlabs/python-predictor-cpu-slim:master
+FROM quay.io/cortexlabs/python-predictor-cpu-slim:master
 
 RUN apt-get update \
     && apt-get install -y tree \
@@ -132,7 +132,7 @@ Update your API configuration file to point to your image:
   ...
 ```
 
-*Note: for [TensorFlow Predictors](#tensorflow-predictor), two containers run together to serve predictions: one runs your Predictor code (`cortexlabs/tensorflow-predictor`), and the other is TensorFlow serving to load the SavedModel (`cortexlabs/tensorflow-serving-gpu` or `cortexlabs/tensorflow-serving-cpu`). There's a second available field `tensorflow_serving_image` that can be used to override the TensorFlow Serving image. Both of the default serving images (`cortexlabs/tensorflow-serving-gpu` and `cortexlabs/tensorflow-serving-cpu`) are based on the official TensorFlow Serving image (`tensorflow/serving`). Unless a different version of TensorFlow Serving is required, the TensorFlow Serving image shouldn't have to be overridden, since it's only used to load the SavedModel and does not run your Predictor code.*
+*Note: for [TensorFlow Predictors](#tensorflow-predictor), two containers run together to serve predictions: one runs your Predictor code (`quay.io/cortexlabs/tensorflow-predictor`), and the other is TensorFlow serving to load the SavedModel (`quay.io/cortexlabs/tensorflow-serving-gpu` or `quay.io/cortexlabs/tensorflow-serving-cpu`). There's a second available field `tensorflow_serving_image` that can be used to override the TensorFlow Serving image. Both of the default serving images (`quay.io/cortexlabs/tensorflow-serving-gpu` and `quay.io/cortexlabs/tensorflow-serving-cpu`) are based on the official TensorFlow Serving image (`tensorflow/serving`). Unless a different version of TensorFlow Serving is required, the TensorFlow Serving image shouldn't have to be overridden, since it's only used to load the SavedModel and does not run your Predictor code.*
 
 Deploy your API as usual:
 
diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go
@@ -62,5 +62,5 @@ func defaultDockerImage(imageName string) string {
 		return fmt.Sprintf("%s/%s:latest", imageOverride, imageName)
 	}
 
-	return fmt.Sprintf("cortexlabs/%s:%s", imageName, CortexVersion)
+	return fmt.Sprintf("quay.io/cortexlabs/%s:%s", imageName, CortexVersion)
 }
diff --git a/pkg/lib/configreader/validators.go b/pkg/lib/configreader/validators.go
@@ -152,7 +152,7 @@ func DurationParser(v *DurationValidation) func(string) (interface{}, error) {
 }
 
 func ValidateImageVersion(image, cortexVersion string) (string, error) {
-	if !strings.HasPrefix(image, "cortexlabs/") && !strings.HasPrefix(image, "cortexlabsdev/") {
+	if !strings.HasPrefix(image, "quay.io/cortexlabs/") && !strings.HasPrefix(image, "quay.io/cortexlabsdev/") && !strings.HasPrefix(image, "cortexlabs/") && !strings.HasPrefix(image, "cortexlabsdev/") {
 		return image, nil
 	}
 
diff --git a/pkg/types/clusterconfig/clusterconfig.go b/pkg/types/clusterconfig/clusterconfig.go
@@ -353,91 +353,91 @@ var UserValidation = &cr.StructValidation{
 		{
 			StructField: "ImageOperator",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/operator:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/operator:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageManager",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/manager:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/manager:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageDownloader",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/downloader:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/downloader:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageRequestMonitor",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/request-monitor:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/request-monitor:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageClusterAutoscaler",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/cluster-autoscaler:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/cluster-autoscaler:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageMetricsServer",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/metrics-server:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/metrics-server:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageInferentia",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/inferentia:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/inferentia:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageNeuronRTD",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/neuron-rtd:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/neuron-rtd:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageNvidia",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/nvidia:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/nvidia:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageFluentd",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/fluentd:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/fluentd:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageStatsd",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/statsd:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/statsd:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageIstioProxy",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/istio-proxy:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/istio-proxy:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
 		{
 			StructField: "ImageIstioPilot",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/istio-pilot:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/istio-pilot:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},
@@ -493,7 +493,7 @@ var AccessValidation = &cr.StructValidation{
 		{
 			StructField: "ImageManager",
 			StringValidation: &cr.StringValidation{
-				Default:   "cortexlabs/manager:" + consts.CortexVersion,
+				Default:   "quay.io/cortexlabs/manager:" + consts.CortexVersion,
 				Validator: validateImageVersion,
 			},
 		},

Original file line number	Diff line number	Diff line change
`@@ -62,5 +62,5 @@ func defaultDockerImage(imageName string) string {`
`62`	`62`	`return fmt.Sprintf("%s/%s:latest", imageOverride, imageName)`
`63`	`63`	`}`
`64`	`64`
`65`		`- return fmt.Sprintf("cortexlabs/%s:%s", imageName, CortexVersion)`
	`65`	`+ return fmt.Sprintf("quay.io/cortexlabs/%s:%s", imageName, CortexVersion)`
`66`	`66`	`}`
Original file line number	Diff line number	Diff line change
`@@ -152,7 +152,7 @@ func DurationParser(v *DurationValidation) func(string) (interface{}, error) {`
`152`	`152`	`}`
`153`	`153`
`154`	`154`	`func ValidateImageVersion(image, cortexVersion string) (string, error) {`
`155`		`- if !strings.HasPrefix(image, "cortexlabs/") && !strings.HasPrefix(image, "cortexlabsdev/") {`
	`155`	`+ if !strings.HasPrefix(image, "quay.io/cortexlabs/") && !strings.HasPrefix(image, "quay.io/cortexlabsdev/") && !strings.HasPrefix(image, "cortexlabs/") && !strings.HasPrefix(image, "cortexlabsdev/") {`
`156`	`156`	`return image, nil`
`157`	`157`	`}`
`158`	`158`