Improve error messages and improve stacktraces (#2374)

Author: vishalbollu

cli/cluster/errors.go

@@ -63,6 +63,7 @@ func ErrorFailedToConnectOperator(originalError error, envName string, operatorU
 		msg += "\nif you have a cluster running:\n"
 		msg += fmt.Sprintf("    → run `cortex cluster info --configure-env %s` to update your environment (include `--config <cluster.yaml>` if you have a cluster configuration file)\n", envName)
 		msg += fmt.Sprintf("    → if you set `operator_load_balancer_scheme: internal` in your cluster configuration file, your CLI must run from within a VPC that has access to your cluster's VPC (see https://docs.cortex.dev/v/%s/)\n", consts.CortexVersionMinor)
+		msg += fmt.Sprintf("    → confirm that the ip address of this machine falls within the CIDR ranges specified in `operator_load_balancer_cidr_whitelist`")
 	}
 
 	return errors.WithStack(&errors.Error{

cli/cmd/cluster.go

@@ -235,14 +235,14 @@ var _clusterUpCmd = &cobra.Command{
 			exit.Error(err)
 		}
 		if exitCode == nil || *exitCode != 0 {
-			out = filterEKSCTLOutput(out)
+			out = s.LastNChars(filterEKSCTLOutput(out), 8192) // get the last 8192 characters because that is the sentry message limit
 			eksCluster, err := awsClient.EKSClusterOrNil(clusterConfig.ClusterName)
 			if err != nil {
 				helpStr := "\ndebugging tips (may or may not apply to this error):"
 				helpStr += fmt.Sprintf("\n* if your cluster started spinning up but was unable to provision instances, additional error information may be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \"Activity\" or \"Activity History\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:", clusterConfig.Region)
 				helpStr += "\n* if your cluster started spinning up, please run `cortex cluster down` to delete the cluster before trying to create this cluster again"
 				fmt.Println(helpStr)
-				exit.Error(ErrorClusterUp(out + helpStr))
+				exit.Error(ErrorClusterUp(out))
 			}
 
 			// the cluster never started spinning up
@@ -398,6 +398,8 @@ var _clusterConfigureCmd = &cobra.Command{
 			exit.Error(err)
 		}
 		if exitCode == nil || *exitCode != 0 {
+			out = s.LastNChars(out, 8192) // get the last 8192 characters because that is the sentry message limit
+
 			helpStr := "\ndebugging tips (may or may not apply to this error):"
 			helpStr += fmt.Sprintf(
 				"\n* if your cluster was unable to provision/remove/scale some nodegroups, additional error information may be found in the description of your cloudformation stack (https://console.aws.amazon.com/cloudformation/home?region=%s#/stacks)"+

pkg/operator/endpoints/errors.go

@@ -53,6 +53,17 @@ func ErrorHeaderMissing(header string) error {
 	})
 }
 
+func ErrorAuthHeaderMissing(header, host, url string) error {
+	return errors.WithStack(&errors.Error{
+		Kind:    ErrHeaderMissing,
+		Message: fmt.Sprintf("missing %s header", header),
+		Metadata: map[string]string{
+			"host": host,
+			"url":  url,
+		},
+	})
+}
+
 func ErrorHeaderMalformed(header string) error {
 	return errors.WithStack(&errors.Error{
 		Kind:    ErrHeaderMalformed,

pkg/operator/endpoints/middleware.go

@@ -64,7 +64,7 @@ func AWSAuthMiddleware(next http.Handler) http.Handler {
 		authHeader := r.Header.Get(consts.AuthHeader)
 
 		if authHeader == "" {
-			respondError(w, r, ErrorHeaderMissing(consts.AuthHeader))
+			respondError(w, r, ErrorAuthHeaderMissing(consts.AuthHeader, r.Host, r.RequestURI))
 			return
 		}
 

pkg/operator/resources/asyncapi/queue_metrics.go

@@ -24,6 +24,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/sqs"
 	"github.com/cortexlabs/cortex/pkg/config"
+	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	"github.com/cortexlabs/cortex/pkg/types/userconfig"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -74,20 +75,20 @@ func updateQueueLengthMetricsFn(apiName, queueURL string) func() error {
 
 		output, err := sqsClient.GetQueueAttributesWithContext(ctx, input)
 		if err != nil {
-			return err
+			return errors.WithStack(err)
 		}
 
 		visibleMessagesStr := output.Attributes["ApproximateNumberOfMessages"]
 		invisibleMessagesStr := output.Attributes["ApproximateNumberOfMessagesNotVisible"]
 
 		visibleMessages, err := strconv.ParseFloat(*visibleMessagesStr, 64)
 		if err != nil {
-			return err
+			return errors.WithStack(err)
 		}
 
 		invisibleMessages, err := strconv.ParseFloat(*invisibleMessagesStr, 64)
 		if err != nil {
-			return err
+			return errors.WithStack(err)
 		}
 
 		activeGauge.WithLabelValues(apiName).Set(invisibleMessages)

python/client/cortex/client.py

@@ -15,31 +15,29 @@
 import json
 import os
 import shutil
-import subprocess
-import sys
-import threading
 import time
 import yaml
-from pathlib import Path
-from typing import Optional, List, Dict, Any
+from typing import List, Dict, Any
 
 from cortex import util
-from cortex.binary import run_cli, get_cli_path
+from cortex.binary import run_cli
 from cortex.telemetry import sentry_wrapper
 
 
 class Client:
     @sentry_wrapper
-    def __init__(self, env: Dict):
+    def __init__(self, env_config: Dict):
         """
         A client to deploy and manage APIs in the specified environment.
+        This constructor is not meant to be invoked directly.
+        Use `cortex.client()` and `cortex.new_client()` to initialize a new cortex client.
 
         Args:
-            env: Environment config
+            env_config: Environment config
         """
 
-        self.env = env
-        self.env_name = env["name"]
+        self.env = env_config
+        self.env_name = env_config["name"]
 
     # CORTEX_VERSION_MINOR
     @sentry_wrapper