Add cloudwatch dashboard (#1054)

Author: tthebst

cli/cmd/cluster.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/cortexlabs/cortex/cli/cluster"
 	"github.com/cortexlabs/cortex/cli/types/cliconfig"
+	"github.com/cortexlabs/cortex/pkg/consts"
 	"github.com/cortexlabs/cortex/pkg/lib/aws"
 	cr "github.com/cortexlabs/cortex/pkg/lib/configreader"
 	"github.com/cortexlabs/cortex/pkg/lib/console"
@@ -155,6 +156,11 @@ var _upCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
+		err = createDashboard(awsClient, clusterConfig.ClusterName)
+		if err != nil {
+			exit.Error(err)
+		}
+
 		out, exitCode, err := runManagerUpdateCommand("/root/install.sh", clusterConfig, awsCreds, _flagClusterEnv)
 		if err != nil {
 			exit.Error(err)
@@ -313,6 +319,11 @@ var _downCmd = &cobra.Command{
 			prompt.YesOrExit(fmt.Sprintf("your cluster named \"%s\" in %s will be spun down and all apis will be deleted, are you sure you want to continue?", *accessConfig.ClusterName, *accessConfig.Region), "", "")
 		}
 
+		err = awsClient.DeleteDashboard(*accessConfig.ClusterName)
+		if err != nil {
+			exit.Error(err)
+		}
+
 		out, exitCode, err := runManagerAccessCommand("/root/uninstall.sh", *accessConfig, awsCreds, _flagClusterEnv)
 		if err != nil {
 			exit.Error(err)
@@ -723,3 +734,26 @@ func CreateLogGroupIfNotFound(awsClient *aws.Client, logGroup string) error {
 
 	return nil
 }
+
+// createDashboard creates a new dashboard (or clears an existing one if it already exists)
+func createDashboard(awsClient *aws.Client, dashboardName string) error {
+	dashboardFound, err := awsClient.DoesDashboardExist(dashboardName)
+	if err != nil {
+		return err
+	}
+
+	if dashboardFound {
+		fmt.Print("￮ using existing cloudwatch dashboard: ", dashboardName)
+	} else {
+		fmt.Print("￮ creating cloudwatch dashboard: ", dashboardName)
+	}
+
+	err = awsClient.CreateDashboard(dashboardName, consts.DashboardTitle)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println(" ✓")
+
+	return nil
+}

cli/cmd/get.go

@@ -323,6 +323,11 @@ func getAPI(env cliconfig.Environment, apiName string) (string, error) {
 	if env.Provider == types.AWSProviderType {
 		apiEndpoint = strings.Replace(urls.Join(apiRes.BaseURL, *api.Endpoint), "https://", "http://", 1)
 	}
+
+	if apiRes.DashboardURL != "" {
+		out += "\n" + console.Bold("metrics dashboard: ") + apiRes.DashboardURL + "\n"
+	}
+
 	out += "\n" + console.Bold("endpoint: ") + apiEndpoint
 
 	out += fmt.Sprintf("\n%s curl %s -X POST -H \"Content-Type: application/json\" -d @sample.json\n", console.Bold("curl:"), apiEndpoint)

docs/guides/metrics.md

@@ -0,0 +1,35 @@
+# Plot in-flight requests
+
+_WARNING: you are on the master branch, please refer to the docs on the branch that matches your `cortex version`_
+
+The `cortex get` and `cortex get API_NAME` commands display the request time (averaged over the past 2 weeks) and response code counts (summed over the past 2 weeks) for your API(s):
+
+```text
+$ cx get
+
+env   api               status   up-to-date   requested   last update   avg request   2XX
+aws   iris-classifier   live     1            1           17m           24ms          1223
+aws   text-generator    live     1            1           8m            180ms         433
+```
+
+The `cortex get API_NAME` command also provides a link to a CloudWatch Metrics dashboard containing this information:
+
+![dashboard](https://user-images.githubusercontent.com/808475/82497902-ed4fd400-9aa3-11ea-8280-20dc6430215f.png)
+
+**responses per minute**
+
+Shows the number of 2XX, 4XX, and 5XX responses per minute.
+
+**total in-flight requests**
+
+Shows the total number of in-flight requests in the cluster.
+
+Note: This is a sum over 10 second intervals because each replica reports it's in-flight requests once per 10 seconds. This plot is only available for the last 3 hours (because second-granular data is aggregated to minute-granular data after 3 hours). To plot data older than 3 hours, instead sum over 1 minute, and divide the y-axis by 6 to determine the number of in-flight requests (since the metrics are reported every 10 seconds).
+
+**median response time**
+
+Shows the median response time for requests, over 1-minute periods (measured in milliseconds).
+
+**p99 response time**
+
+Shows the p99 response time for requests, over 1-minute periods (measured in milliseconds).

docs/guides/plot-in-flight-requests.md

@@ -47,11 +47,9 @@
 
 ## Guides
 
+* [View API metrics](guides/metrics.md)
 * [Set up AWS API gateway](guides/api-gateway.md)
 * [Set up HTTPS on a subdomain](guides/subdomain-https-setup.md)
-* [Plot response code counts](guides/plot-response-code-counts.md)
-* [Plot API request time](guides/plot-request-time.md)
-* [Plot in-flight requests](guides/plot-in-flight-requests.md)
 * [Set up VPC peering](guides/vpc-peering.md)
 * [Add a batch runner API](guides/batch-runner.md)
 * [SSH into worker instance](guides/ssh-instance.md)

docs/guides/plot-request-time.md

@@ -45,6 +45,7 @@ var (
 	)
 
 	MaxClassesPerMonitoringRequest = 20 // cloudwatch.GeMetricData can get up to 100 metrics per request, avoid multiple requests and have room for other stats
+	DashboardTitle                 = "# cortex monitoring dashboard"
 )
 
 func defaultDockerImage(imageName string) string {

docs/guides/plot-response-code-counts.md

@@ -32,19 +32,19 @@ import (
 )
 
 type clients struct {
-	s3                *s3.S3
-	s3Uploader        *s3manager.Uploader
-	s3Downloader      *s3manager.Downloader
-	sts               *sts.STS
-	ec2               *ec2.EC2
-	ecr               *ecr.ECR
-	acm               *acm.ACM
-	autoscaling       *autoscaling.AutoScaling
-	cloudWatchLogs    *cloudwatchlogs.CloudWatchLogs
-	cloudWatchMetrics *cloudwatch.CloudWatch
-	serviceQuotas     *servicequotas.ServiceQuotas
-	cloudFormation    *cloudformation.CloudFormation
-	iam               *iam.IAM
+	s3             *s3.S3
+	s3Uploader     *s3manager.Uploader
+	s3Downloader   *s3manager.Downloader
+	sts            *sts.STS
+	ec2            *ec2.EC2
+	ecr            *ecr.ECR
+	acm            *acm.ACM
+	autoscaling    *autoscaling.AutoScaling
+	cloudWatchLogs *cloudwatchlogs.CloudWatchLogs
+	cloudWatch     *cloudwatch.CloudWatch
+	serviceQuotas  *servicequotas.ServiceQuotas
+	cloudFormation *cloudformation.CloudFormation
+	iam            *iam.IAM
 }
 
 func (c *Client) S3() *s3.S3 {
@@ -117,11 +117,11 @@ func (c *Client) CloudWatchLogs() *cloudwatchlogs.CloudWatchLogs {
 	return c.clients.cloudWatchLogs
 }
 
-func (c *Client) CloudWatchMetrics() *cloudwatch.CloudWatch {
-	if c.clients.cloudWatchMetrics == nil {
-		c.clients.cloudWatchMetrics = cloudwatch.New(c.sess)
+func (c *Client) CloudWatch() *cloudwatch.CloudWatch {
+	if c.clients.cloudWatch == nil {
+		c.clients.cloudWatch = cloudwatch.New(c.sess)
 	}
-	return c.clients.cloudWatchMetrics
+	return c.clients.cloudWatch
 }
 
 func (c *Client) ServiceQuotas() *servicequotas.ServiceQuotas {