BatchJob CRD (#2038)

Author: Miguel Varela Ramos

.circleci/config.yml

@@ -87,6 +87,25 @@ jobs:
             go get -u -v golang.org/x/lint/golint
             go get -u -v github.com/kyoh86/looppointer/cmd/looppointer
             sudo pip install black aiohttp
+      - run:
+          name: Initialize Credentials
+          command: |
+            echo 'export AWS_ACCESS_KEY_ID=${NIGHTLY_AWS_ACCESS_KEY_ID}' >> $BASH_ENV
+            echo 'export AWS_SECRET_ACCESS_KEY=${NIGHTLY_AWS_SECRET_ACCESS_KEY}' >> $BASH_ENV
+      - run:
+          name: Generate Cluster Config
+          command: |
+            mkdir -p dev/config
+            cat \<< EOF > ./dev/config/cluster.yaml
+            cluster_name: cortex
+            region: us-east-1
+            bucket: cortex-dev-nightly
+            node_groups:
+              - name: cpu
+                instance_type: m5.large
+                min_instances: 1
+                max_instances: 1
+            EOF
       - run:
           name: Lint
           command: make lint

.dockerignore

@@ -12,3 +12,7 @@
 **/*.pyo
 **/*.pyd
 **/__pycache__/
+
+**/hack/
+**/PROJECT
+**/Makefile

.gitignore

@@ -1,5 +1,6 @@
 /vendor/
 /bin/
+/testbin/
 /dev/config/
 
 # PYTHON
@@ -20,3 +21,23 @@ __pycache__/
 *.zip
 .unison*
 .vscode/
+
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Kubernetes Generated files - skip generated files, except for vendored files
+
+!vendor/**/zz_generated.*
+
+# editor and IDE paraphernalia
+.idea
+*.swp
+*.swo
+*~

build/images.sh

@@ -31,13 +31,15 @@ dev_images=(
   "manager"
   "request-monitor"
   "async-gateway"
+  "enqueuer"
 )
 
 non_dev_images=(
   "tensorflow-serving-cpu"
   "tensorflow-serving-gpu"
   "cluster-autoscaler"
   "operator"
+  "controller-manager"
   "istio-proxy"
   "istio-pilot"
   "fluent-bit"
@@ -56,6 +58,7 @@ non_dev_images=(
   "inferentia"
   "neuron-rtd"
   "nvidia"
+  "kubexit"
 )
 
 all_images=(

build/lint.sh

@@ -89,9 +89,11 @@ output=$(cd "$ROOT" && find . -type f \
 ! -path "**/*.egg-info/*" \
 ! -path "./test/*" \
 ! -path "./dev/config/*" \
-! -path "./bin/*" \
+! -path "**/bin/*" \
 ! -path "./.circleci/*" \
 ! -path "./.git/*" \
+! -path "./pkg/crds/config/*" \
+! -path "**/tmp/*" \
 ! -name LICENSE \
 ! -name "*requirements.txt" \
 ! -name "go.*" \
@@ -100,6 +102,7 @@ output=$(cd "$ROOT" && find . -type f \
 ! -name ".*" \
 ! -name "*.bin" \
 ! -name "Dockerfile" \
+! -name "PROJECT" \
 -exec grep -L "Copyright 2021 Cortex Labs, Inc" {} \;)
 if [[ $output ]]; then
   echo "File(s) are missing Cortex license:"
@@ -119,7 +122,7 @@ if [ "$is_release_branch" = "true" ]; then
   ! -path "**/.pytest_cache/*" \
   ! -path "**/*.egg-info/*" \
   ! -path "./dev/config/*" \
-  ! -path "./bin/*" \
+  ! -path "**/bin/*" \
   ! -path "./.git/*" \
   ! -name ".*" \
   ! -name "*.bin" \
@@ -142,8 +145,9 @@ output=$(cd "$ROOT" && find . -type f \
 ! -path "**/.pytest_cache/*" \
 ! -path "**/*.egg-info/*" \
 ! -path "./dev/config/*" \
-! -path "./bin/*" \
+! -path "**/bin/*" \
 ! -path "./.git/*" \
+! -path "./pkg/crds/config/*" \
 ! -name ".*" \
 ! -name "*.bin" \
 ! -name "*.wav" \
@@ -164,7 +168,8 @@ output=$(cd "$ROOT" && find . -type f \
 ! -path "**/.pytest_cache/*" \
 ! -path "**/*.egg-info/*" \
 ! -path "./dev/config/*" \
-! -path "./bin/*" \
+! -path "./pkg/crds/config/*" \
+! -path "**/bin/*" \
 ! -path "./.git/*" \
 ! -name ".*" \
 ! -name "*.bin" \
@@ -187,7 +192,7 @@ output=$(cd "$ROOT" && find . -type f \
 ! -path "**/.pytest_cache/*" \
 ! -path "**/*.egg-info/*" \
 ! -path "./dev/config/*" \
-! -path "./bin/*" \
+! -path "**/bin/*" \
 ! -path "./.git/*" \
 ! -name ".*" \
 ! -name "*.bin" \
@@ -209,11 +214,13 @@ output=$(cd "$ROOT" && find . -type f \
 ! -path "**/.pytest_cache/*" \
 ! -path "**/*.egg-info/*" \
 ! -path "./dev/config/*" \
+! -path "./pkg/crds/config/*" \
 ! -path "./bin/*" \
 ! -path "./.git/*" \
 ! -name ".*" \
 ! -name "*.bin" \
 ! -name "*.wav" \
+! -name "boilerplate.go.txt" \
 -print0 | \
 xargs -0 -L1 bash -c 'test "$(head -c 1 "$0")" || [ ! -s "$0" ] || echo "New line at beginning of $0"' || true)
 if [[ $output ]]; then

build/test.sh

@@ -18,6 +18,7 @@
 set -euo pipefail
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
+ENVTEST_ASSETS_DIR=${ROOT}/testbin
 
 cluster_env="undefined"
 create_cluster="no"
@@ -69,7 +70,13 @@ cmd=${1:-""}
 sub_cmd=${2:-""}
 
 function run_go_tests() {
-  (cd $ROOT && go test ./... && echo "go tests passed")
+  (
+    cd $ROOT
+    mkdir -p ${ENVTEST_ASSETS_DIR}
+	  test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh
+	  source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools ${ENVTEST_ASSETS_DIR}; setup_envtest_env ${ENVTEST_ASSETS_DIR}
+    go test ./... && echo "go tests passed"
+  )
 }
 
 function run_python_tests() {

docs/clusters/management/create.md

@@ -97,6 +97,7 @@ The docker images used by the cluster can also be overridden. They can be config
 <!-- CORTEX_VERSION_BRANCH_STABLE -->
 ```yaml
 image_operator: quay.io/cortexlabs/operator:master
+image_controller_manager: quay.io/cortexlabs/controller-manager:master
 image_manager: quay.io/cortexlabs/manager:master
 image_downloader: quay.io/cortexlabs/downloader:master
 image_request_monitor: quay.io/cortexlabs/request-monitor:master
@@ -119,4 +120,6 @@ image_prometheus_node_exporter: quay.io/cortexlabs/prometheus-node-exporter:mast
 image_kube_rbac_proxy: quay.io/cortexlabs/kube-rbac-proxy:master
 image_grafana: quay.io/cortexlabs/grafana:master
 image_event_exporter: quay.io/cortexlabs/event-exporter:master
+image_enqueuer: quay.io/cortexlabs/enqueuer:master
+image_kubexit: quay.io/cortexlabs/kubexit:master
 ```