Wait for load balancers to be ready when creating cluster (#1431)

Author: deliahu

manager/debug.sh

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set +e
+
 CORTEX_VERSION_MINOR=master
 
 debug_out_path="$1"
@@ -31,21 +33,35 @@ echo -n "gathering cluster data"
 
 mkdir -p /cortex-debug/k8s
 for resource in pods pods.metrics nodes nodes.metrics daemonsets deployments hpa services virtualservices gateways ingresses configmaps jobs replicasets events; do
-  kubectl describe $resource --all-namespaces &>/dev/null > "/cortex-debug/k8s/${resource}"
-  kubectl get $resource --all-namespaces &>/dev/null > "/cortex-debug/k8s/${resource}-list"
+  kubectl describe $resource --all-namespaces > "/cortex-debug/k8s/${resource}" 2>&1
+  kubectl get $resource --all-namespaces > "/cortex-debug/k8s/${resource}-list" 2>&1
   echo -n "."
 done
 
 mkdir -p /cortex-debug/logs
-kubectl get pods --all-namespaces -o json | jq '.items[] | "kubectl logs -n \(.metadata.namespace) \(.metadata.name) --all-containers --timestamps --tail=10000 &>/dev/null > /cortex-debug/logs/\(.metadata.namespace).\(.metadata.name) && echo -n ."' | xargs -n 1 bash -c
+kubectl get pods --all-namespaces -o json | jq '.items[] | "kubectl logs -n \(.metadata.namespace) \(.metadata.name) --all-containers --timestamps --tail=10000 > /cortex-debug/logs/\(.metadata.namespace).\(.metadata.name) 2>&1 && echo -n ."' | xargs -n 1 bash -c
 
-kubectl top pods --all-namespaces --containers=true &>/dev/null > "/cortex-debug/k8s/top_pods"
-kubectl top nodes &>/dev/null > "/cortex-debug/k8s/top_nodes"
+kubectl top pods --all-namespaces --containers=true > "/cortex-debug/k8s/top_pods" 2>&1
+kubectl top nodes > "/cortex-debug/k8s/top_nodes" 2>&1
 
 mkdir -p /cortex-debug/aws
-aws --region=$CORTEX_REGION autoscaling describe-auto-scaling-groups &>/dev/null > "/cortex-debug/aws/asgs"
+aws --region=$CORTEX_REGION autoscaling describe-auto-scaling-groups > "/cortex-debug/aws/asgs" 2>&1
+echo -n "."
+aws --region=$CORTEX_REGION autoscaling describe-scaling-activities > "/cortex-debug/aws/asg-activities" 2>&1
+echo -n "."
+python get_operator_load_balancer_state.py > "/cortex-debug/aws/operator_load_balancer_state" 2>&1
+python get_api_load_balancer_state.py > "/cortex-debug/aws/api_load_balancer_state" 2>&1
+python get_operator_target_group_status.py > "/cortex-debug/aws/operator_load_balancer_target_group_status" 2>&1
 echo -n "."
-aws --region=$CORTEX_REGION autoscaling describe-scaling-activities &>/dev/null > "/cortex-debug/aws/asg-activities"
+
+mkdir -p /cortex-debug/misc
+operator_endpoint=$(kubectl -n=istio-system get service ingressgateway-operator -o json 2>/dev/null | tr -d '[:space:]' | sed 's/.*{\"hostname\":\"\(.*\)\".*/\1/')
+echo "$operator_endpoint" > /cortex-debug/misc/operator_endpoint
+if [ "$operator_endpoint" == "" ]; then
+  echo "unable to get operator endpoint" > /cortex-debug/misc/operator_curl
+else
+  curl -sv --max-time 5 "${operator_endpoint}/verifycortex" > /cortex-debug/misc/operator_curl 2>&1
+fi
 echo -n "."
 
 (cd / && tar -czf cortex-debug.tgz cortex-debug)

manager/get_api_load_balancer_state.py

@@ -0,0 +1,32 @@
+# Copyright 2020 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import boto3
+import os
+
+from helpers import get_api_load_balancer
+
+
+def get_api_load_balancer_state():
+    cluster_name = os.environ["CORTEX_CLUSTER_NAME"]
+    region = os.environ["CORTEX_REGION"]
+
+    client_elbv2 = boto3.client("elbv2", region_name=region)
+
+    load_balancer = get_api_load_balancer(cluster_name, client_elbv2)
+    return load_balancer["State"]["Code"]
+
+
+if __name__ == "__main__":
+    print(get_api_load_balancer_state(), end="")

manager/get_operator_load_balancer_state.py

@@ -0,0 +1,32 @@
+# Copyright 2020 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import boto3
+import os
+
+from helpers import get_operator_load_balancer
+
+
+def get_operator_load_balancer_state():
+    cluster_name = os.environ["CORTEX_CLUSTER_NAME"]
+    region = os.environ["CORTEX_REGION"]
+
+    client_elbv2 = boto3.client("elbv2", region_name=region)
+
+    load_balancer = get_operator_load_balancer(cluster_name, client_elbv2)
+    return load_balancer["State"]["Code"]
+
+
+if __name__ == "__main__":
+    print(get_operator_load_balancer_state(), end="")

manager/get_operator_target_group_status.py

@@ -0,0 +1,55 @@
+# Copyright 2020 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import boto3
+import os
+import json
+
+from helpers import get_operator_load_balancer
+
+
+def get_operator_target_group_status():
+    cluster_name = os.environ["CORTEX_CLUSTER_NAME"]
+    region = os.environ["CORTEX_REGION"]
+
+    client_elbv2 = boto3.client("elbv2", region_name=region)
+
+    load_balancer_arn = get_operator_load_balancer(cluster_name, client_elbv2)["LoadBalancerArn"]
+    target_group_arn = get_load_balancer_https_target_group_arn(load_balancer_arn, client_elbv2)
+    return get_target_health(target_group_arn, client_elbv2)
+
+
+def get_load_balancer_https_target_group_arn(load_balancer_arn, client_elbv2):
+    paginator = client_elbv2.get_paginator("describe_listeners")
+    for listener_page in paginator.paginate(LoadBalancerArn=load_balancer_arn):
+        for listener in listener_page["Listeners"]:
+            if listener["Port"] == 443:
+                return listener["DefaultActions"][0]["TargetGroupArn"]
+
+    raise Exception(
+        f"unable to find https target group for operator load balancer ({load_balancer_arn})"
+    )
+
+
+def get_target_health(target_group_arn, client_elbv2):
+    response = client_elbv2.describe_target_health(TargetGroupArn=target_group_arn)
+    for health_description in response["TargetHealthDescriptions"]:
+        if health_description["TargetHealth"]["State"] == "healthy":
+            return "healthy"
+
+    return json.dumps(response["TargetHealthDescriptions"])
+
+
+if __name__ == "__main__":
+    print(get_operator_target_group_status(), end="")

manager/helpers.py

@@ -0,0 +1,45 @@
+# Copyright 2020 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def get_operator_load_balancer(cluster_name, client_elbv2):
+    return _get_load_balancer("operator", cluster_name, client_elbv2)
+
+
+def get_api_load_balancer(cluster_name, client_elbv2):
+    return _get_load_balancer("api", cluster_name, client_elbv2)
+
+
+def _get_load_balancer(load_balancer_tag, cluster_name, client_elbv2):
+    paginator = client_elbv2.get_paginator("describe_load_balancers")
+    for load_balancer_page in paginator.paginate(PaginationConfig={"PageSize": 20}):
+        load_balancers = {
+            load_balancer["LoadBalancerArn"]: load_balancer
+            for load_balancer in load_balancer_page["LoadBalancers"]
+        }
+        tag_descriptions = client_elbv2.describe_tags(ResourceArns=list(load_balancers.keys()))[
+            "TagDescriptions"
+        ]
+        for tag_description in tag_descriptions:
+            foundClusterNameTag = False
+            foundLoadBalancerTag = False
+            for tags in tag_description["Tags"]:
+                if tags["Key"] == "cortex.dev/cluster-name" and tags["Value"] == cluster_name:
+                    foundClusterNameTag = True
+                if tags["Key"] == "cortex.dev/load-balancer" and tags["Value"] == load_balancer_tag:
+                    foundLoadBalancerTag = True
+            if foundClusterNameTag and foundLoadBalancerTag:
+                return load_balancers[tag_description["ResourceArn"]]
+
+    raise Exception(f"unable to find {load_balancer_tag} load balancer")