Add commands for testing apis (#1763)

Author: RobertLucian

Makefile

@@ -239,8 +239,9 @@ tools:
 	@go get -u -v golang.org/x/lint/golint
 	@go get -u -v github.com/VojtechVitek/rerun/cmd/rerun
 	@go get -u -v github.com/go-delve/delve/cmd/dlv
-	@python3 -m pip install aiohttp black 'pydoc-markdown>=3.0.0,<4.0.0'
 	@if [[ "$$OSTYPE" == "darwin"* ]]; then brew install parallel; elif [[ "$$OSTYPE" == "linux"* ]]; then sudo apt-get install -y parallel; else echo "your operating system is not supported"; fi
+	@python3 -m pip install aiohttp black 'pydoc-markdown>=3.0.0,<4.0.0'
+	@python3 -m pip install -e test/e2e
 
 format:
 	@./dev/format.sh
@@ -258,6 +259,26 @@ test-go:
 test-python:
 	@./build/test.sh python
 
+# run e2e tests on existing cluster
+# read test/e2e/README.md for instructions first
+test-e2e:
+	@$(MAKE) test-e2e-aws
+	@$(MAKE) test-e2e-gcp
+test-e2e-aws:
+	@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster-aws.yaml) && CORTEX_CLI_PATH="$$(pwd)/bin/cortex" ./build/test.sh e2e -p aws -e "$$CORTEX_CLUSTER_NAME-aws"
+test-e2e-gcp:
+	@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster-gcp.yaml) && CORTEX_CLI_PATH="$$(pwd)/bin/cortex" ./build/test.sh e2e -p gcp -e "$$CORTEX_CLUSTER_NAME-gcp"
+
+# run e2e tests with new clusters
+# read test/e2e/README.md for instructions first
+test-e2e-new:
+	@$(MAKE) test-e2e-new-aws
+	@$(MAKE) test-e2e-new-gcp
+test-e2e-new-aws:
+	@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster-aws.yaml) && CORTEX_CLI_PATH="$$(pwd)/bin/cortex" ./build/test.sh e2e "$$(pwd)/dev/config/cluster-aws.yaml" -p aws --create-cluster
+test-e2e-new-gcp:
+	@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster-gcp.yaml) && CORTEX_CLI_PATH="$$(pwd)/bin/cortex" ./build/test.sh e2e "$$(pwd)/dev/config/cluster-gcp.yaml" -p gcp --create-cluster
+
 lint:
 	@./build/lint.sh
 

build/test.sh

@@ -19,6 +19,64 @@ set -euo pipefail
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 
+provider="undefined"
+cluster_env="undefined"
+create_cluster="no"
+positional_args=()
+while [[ $# -gt 0 ]]; do
+  key="$1"
+  case $key in
+    -p|--provider)
+    provider="$2"
+    shift
+    ;;
+    -e|--cluster-env)
+    cluster_env="$2"
+    shift
+    ;;
+    -c|--create-cluster)
+    create_cluster="yes"
+    shift
+    ;;
+    *)
+    positional_args+=("$1")
+    shift
+    ;;
+  esac
+done
+set -- "${positional_args[@]}"
+positional_args=()
+for i in "$@"; do
+  case $i in
+    -p=*|--provider=*)
+    provider="${i#*=}"
+    shift
+    ;;
+    -e=*|--cluster-env=*)
+    cluster_env="${i#*=}"
+    shift
+    ;;
+    -c|--create-cluster)
+    create_cluster="yes"
+    shift
+    ;;
+    *)
+    positional_args+=("$1")
+    shift
+    ;;
+  esac
+done
+set -- "${positional_args[@]}"
+for arg in "$@"; do
+  if [[ "$arg" == -* ]]; then
+    echo "unknown flag: $arg"
+    exit 1
+  fi
+done
+
+cmd=${1:-""}
+sub_cmd=${2:-""}
+
 function run_go_tests() {
   (cd $ROOT && go test ./... && echo "go tests passed")
 }
@@ -28,12 +86,28 @@ function run_python_tests() {
   docker run cortexlabs/test
 }
 
-cmd=${1:-""}
+function run_e2e_tests() {
+  if [ "$provider" = "aws" ]; then
+    if [ "$create_cluster" = "yes" ]; then
+      pytest $ROOT/test/e2e/tests -k aws --aws-config "$sub_cmd"
+    else
+      pytest $ROOT/test/e2e/tests -k aws --aws-env "$cluster_env"
+    fi
+  elif [ "$provider" = "gcp" ]; then
+    if [ "$create_cluster" = "yes" ]; then
+      pytest $ROOT/test/e2e/tests -k gcp --gcp-config "$sub_cmd"
+    else
+      pytest $ROOT/test/e2e/tests -k gcp --gcp-env "$cluster_env"
+    fi
+  fi
+}
 
 if [ "$cmd" = "go" ]; then
   run_go_tests
 elif [ "$cmd" = "python" ]; then
   run_python_tests
+elif [ "$cmd" = "e2e" ]; then
+  run_e2e_tests
 else
   run_go_tests
   run_python_tests

test/e2e/e2e/cluster.py

@@ -28,10 +28,15 @@ def create_cluster(cluster_config: str):
     cluster_name = config["cluster_name"]
     provider = config["provider"]
 
+    if provider == "gcp":
+        cluster_cmd = "cluster-gcp"
+    else:
+        cluster_cmd = "cluster"
+
     p = subprocess.run(
         [
             "cortex",
-            "cluster",
+            cluster_cmd,
             "up",
             "-y",
             "--config",
@@ -49,8 +54,16 @@ def create_cluster(cluster_config: str):
 
 def delete_cluster(cluster_config: str):
     """Delete a cortex cluster from a cluster config"""
+    with open(cluster_config) as f:
+        config = yaml.safe_load(f)
+
+    if config["provider"] == "gcp":
+        cluster_cmd = "cluster-gcp"
+    else:
+        cluster_cmd = "cluster"
+
     p = subprocess.run(
-        ["cortex", "cluster", "down", "-y", "--config", cluster_config],
+        ["cortex", cluster_cmd, "down", "-y", "--config", cluster_config],
         stdout=sys.stdout,
         stderr=sys.stderr,
     )