Check EKS status before spinning up cluster (#561)

(cherry picked from commit 9118cf4)

Author: deliahu

manager/install.sh

@@ -19,7 +19,14 @@ set -e
 arg1="$1"
 
 function ensure_eks() {
-  if ! eksctl utils describe-stacks --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION >/dev/null 2>&1; then
+  # Cluster statuses: https://github.com/aws/aws-sdk-go/blob/master/service/eks/api.go#L2785
+  set +e
+  cluster_info=$(eksctl get cluster --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION -o json)
+  cluster_info_exit_code=$?
+  set -e
+
+  # No cluster
+  if [ $cluster_info_exit_code -ne 0 ]; then
     if [ "$arg1" = "--update" ]; then
       echo "error: there isn't a Cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION; please update your configuration to point to an existing Cortex cluster or create a Cortex cluster with \`cortex cluster up\`"
       exit 1
@@ -31,6 +38,29 @@ function ensure_eks() {
     return
   fi
 
+  cluster_status=$(echo "$cluster_info" | jq -r 'first | .Status')
+
+  if [ "$cluster_status" == "DELETING" ]; then
+    echo "error: your Cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION is currently spinning down; please try again once it is completely deleted (may take a few minutes)"
+    exit 1
+  fi
+
+  if [ "$cluster_status" == "CREATING" ]; then
+    echo "error: your Cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION is currently spinning up; please try again once it is ready"
+    exit 1
+  fi
+
+  if [ "$cluster_status" == "FAILED" ]; then
+    echo "error: your Cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION is failed; delete it with \`eksctl delete cluster --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION\` and try again"
+    exit 1
+  fi
+
+  # Catch all
+  if [ "$cluster_status" != "ACTIVE" ]; then
+    echo "error: your Cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION is not active; please wait until it is active, or delete it with \`eksctl delete cluster --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION\` and try again"
+    exit 1
+  fi
+
   echo "✓ Cluster is running"
 
   # Check if instance type changed