Skip to content

Commit 3ac0c14

Browse files
RobertLucianRobertLucian
authored
Improve observability of API statuses (#2368)
1 parent d0c010a commit 3ac0c14

File tree

47 files changed

+1338
-974
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1338
-974
lines changed

‎cli/cluster/delete.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222

2323
"github.com/cortexlabs/cortex/pkg/lib/errors"
2424
"github.com/cortexlabs/cortex/pkg/lib/json"
25+
"github.com/cortexlabs/cortex/pkg/lib/pointer"
2526
"github.com/cortexlabs/cortex/pkg/lib/prompt"
2627
s "github.com/cortexlabs/cortex/pkg/lib/strings"
2728
"github.com/cortexlabs/cortex/pkg/operator/schema"
@@ -70,8 +71,7 @@ func getReadyRealtimeAPIReplicasOrNil(operatorConfig OperatorConfig, apiName str
7071
return nil
7172
}
7273

73-
totalReady := apiRes.Status.Updated.Ready + apiRes.Status.Stale.Ready
74-
return &totalReady
74+
return pointer.Int32(apiRes.Status.Ready)
7575
}
7676

7777
func StopJob(operatorConfig OperatorConfig, kind userconfig.Kind, apiName string, jobID string) (schema.DeleteResponse, error) {

‎cli/cluster/get.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,20 @@ func GetAPI(operatorConfig OperatorConfig, apiName string) ([]schema.APIResponse
5151
return apiRes, nil
5252
}
5353

54+
func DescribeAPI(operatorConfig OperatorConfig, apiName string) ([]schema.APIResponse, error) {
55+
httpRes, err := HTTPGet(operatorConfig, "/describe/"+apiName)
56+
if err != nil {
57+
return nil, err
58+
}
59+
60+
var apiRes []schema.APIResponse
61+
if err = json.Unmarshal(httpRes, &apiRes); err != nil {
62+
return nil, errors.Wrap(err, "/describe/"+apiName, string(httpRes))
63+
}
64+
65+
return apiRes, nil
66+
}
67+
5468
func GetAPIByID(operatorConfig OperatorConfig, apiName string, apiID string) ([]schema.APIResponse, error) {
5569
httpRes, err := HTTPGet(operatorConfig, "/get/"+apiName+"/"+apiID)
5670
if err != nil {

‎cli/cmd/describe.go‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/*
2+
Copyright 2021 Cortex Labs, Inc.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package cmd
18+
19+
import (
20+
"fmt"
21+
22+
"github.com/cortexlabs/cortex/cli/cluster"
23+
"github.com/cortexlabs/cortex/cli/types/cliconfig"
24+
"github.com/cortexlabs/cortex/pkg/lib/errors"
25+
"github.com/cortexlabs/cortex/pkg/lib/exit"
26+
"github.com/cortexlabs/cortex/pkg/lib/telemetry"
27+
"github.com/cortexlabs/cortex/pkg/types/userconfig"
28+
"github.com/spf13/cobra"
29+
)
30+
31+
const (
32+
_titleReplicaStatus = "replica status"
33+
_titleReplicaCount = "replica count"
34+
)
35+
36+
var (
37+
_flagDescribeEnv string
38+
_flagDescribeWatch bool
39+
)
40+
41+
func describeInit() {
42+
_describeCmd.Flags().SortFlags = false
43+
_describeCmd.Flags().StringVarP(&_flagDescribeEnv, "env", "e", "", "environment to use")
44+
_describeCmd.Flags().BoolVarP(&_flagDescribeWatch, "watch", "w", false, "re-run the command every 2 seconds")
45+
}
46+
47+
var _describeCmd = &cobra.Command{
48+
Use: "describe [API_NAME]",
49+
Short: "describe an api",
50+
Args: cobra.ExactArgs(1),
51+
Run: func(cmd *cobra.Command, args []string) {
52+
apiName := args[0]
53+
54+
var envName string
55+
if wasFlagProvided(cmd, "env") {
56+
envName = _flagDescribeEnv
57+
} else {
58+
var err error
59+
envName, err = getEnvFromFlag("")
60+
if err != nil {
61+
telemetry.Event("cli.describe")
62+
exit.Error(err)
63+
}
64+
}
65+
66+
env, err := ReadOrConfigureEnv(envName)
67+
if err != nil {
68+
telemetry.Event("cli.describe")
69+
exit.Error(err)
70+
}
71+
telemetry.Event("cli.describe", map[string]interface{}{"env_name": env.Name})
72+
73+
rerun(_flagDescribeWatch, func() (string, error) {
74+
env, err := ReadOrConfigureEnv(envName)
75+
if err != nil {
76+
exit.Error(err)
77+
}
78+
79+
out, err := envStringIfNotSpecified(envName, cmd)
80+
if err != nil {
81+
return "", err
82+
}
83+
apiTable, err := describeAPI(env, apiName)
84+
if err != nil {
85+
return "", err
86+
}
87+
88+
return out + apiTable, nil
89+
})
90+
},
91+
}
92+
93+
func describeAPI(env cliconfig.Environment, apiName string) (string, error) {
94+
apisRes, err := cluster.DescribeAPI(MustGetOperatorConfig(env.Name), apiName)
95+
if err != nil {
96+
return "", err
97+
}
98+
99+
if len(apisRes) == 0 {
100+
exit.Error(errors.ErrorUnexpected(fmt.Sprintf("unable to find api %s", apiName)))
101+
}
102+
103+
apiRes := apisRes[0]
104+
105+
switch apiRes.Metadata.Kind {
106+
case userconfig.RealtimeAPIKind:
107+
return realtimeDescribeAPITable(apiRes, env)
108+
case userconfig.AsyncAPIKind:
109+
return asyncDescribeAPITable(apiRes, env)
110+
default:
111+
return "", errors.ErrorUnexpected(fmt.Sprintf("encountered unexpected kind %s for api %s", apiRes.Spec.Kind, apiRes.Spec.Name))
112+
}
113+
}

‎cli/cmd/get.go‎

Lines changed: 40 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -35,29 +35,28 @@ import (
3535
libtime "github.com/cortexlabs/cortex/pkg/lib/time"
3636
"github.com/cortexlabs/cortex/pkg/operator/schema"
3737
"github.com/cortexlabs/cortex/pkg/types/userconfig"
38+
"github.com/cortexlabs/yaml"
3839
"github.com/spf13/cobra"
3940
)
4041

4142
const (
4243
_titleEnvironment = "env"
4344
_titleRealtimeAPI = "realtime api"
44-
_titleStatus = "status"
45+
_titleAsyncAPI = "async api"
46+
_titleLive = "live"
4547
_titleUpToDate = "up-to-date"
46-
_titleStale = "stale"
47-
_titleRequested = "requested"
48-
_titleFailed = "failed"
49-
_titleLastupdated = "last update"
48+
_titleLastUpdated = "last update"
5049
)
5150

5251
var (
53-
_flagGetEnv string
54-
_flagWatch bool
52+
_flagGetEnv string
53+
_flagGetWatch bool
5554
)
5655

5756
func getInit() {
5857
_getCmd.Flags().SortFlags = false
5958
_getCmd.Flags().StringVarP(&_flagGetEnv, "env", "e", "", "environment to use")
60-
_getCmd.Flags().BoolVarP(&_flagWatch, "watch", "w", false, "re-run the command every 2 seconds")
59+
_getCmd.Flags().BoolVarP(&_flagGetWatch, "watch", "w", false, "re-run the command every 2 seconds")
6160
_getCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), "|")))
6261
addVerboseFlag(_getCmd)
6362
}
@@ -90,7 +89,7 @@ var _getCmd = &cobra.Command{
9089
telemetry.Event("cli.get")
9190
}
9291

93-
rerun(func() (string, error) {
92+
rerun(_flagGetWatch, func() (string, error) {
9493
if len(args) == 1 {
9594
env, err := ReadOrConfigureEnv(envName)
9695
if err != nil {
@@ -106,7 +105,7 @@ var _getCmd = &cobra.Command{
106105
return "", err
107106
}
108107

109-
if _flagOutput == flags.JSONOutputType {
108+
if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {
110109
return apiTable, nil
111110
}
112111

@@ -136,7 +135,7 @@ var _getCmd = &cobra.Command{
136135
if err != nil {
137136
return "", err
138137
}
139-
if _flagOutput == flags.JSONOutputType {
138+
if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {
140139
return jobTable, nil
141140
}
142141

@@ -166,7 +165,7 @@ var _getCmd = &cobra.Command{
166165
return "", err
167166
}
168167

169-
if _flagOutput == flags.JSONOutputType {
168+
if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {
170169
return apiTable, nil
171170
}
172171

@@ -221,7 +220,7 @@ func getAPIsInAllEnvironments() (string, error) {
221220

222221
if err == nil {
223222
for _, api := range apisRes {
224-
switch api.Spec.Kind {
223+
switch api.Metadata.Kind {
225224
case userconfig.BatchAPIKind:
226225
allBatchAPIEnvs = append(allBatchAPIEnvs, env.Name)
227226
allBatchAPIs = append(allBatchAPIs, api)
@@ -247,12 +246,16 @@ func getAPIsInAllEnvironments() (string, error) {
247246
allAPIsOutput = append(allAPIsOutput, apisOutput)
248247
}
249248

249+
var bytes []byte
250250
if _flagOutput == flags.JSONOutputType {
251-
bytes, err := libjson.Marshal(allAPIsOutput)
252-
if err != nil {
253-
return "", err
254-
}
255-
251+
bytes, err = libjson.Marshal(allAPIsOutput)
252+
} else if _flagOutput == flags.YAMLOutputType {
253+
bytes, err = yaml.Marshal(allAPIsOutput)
254+
}
255+
if err != nil {
256+
return "", err
257+
}
258+
if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {
256259
return string(bytes), nil
257260
}
258261

@@ -337,11 +340,16 @@ func getAPIsByEnv(env cliconfig.Environment) (string, error) {
337340
return "", err
338341
}
339342

343+
var bytes []byte
340344
if _flagOutput == flags.JSONOutputType {
341-
bytes, err := libjson.Marshal(apisRes)
342-
if err != nil {
343-
return "", err
344-
}
345+
bytes, err = libjson.Marshal(apisRes)
346+
} else if _flagOutput == flags.YAMLOutputType {
347+
bytes, err = yaml.Marshal(apisRes)
348+
}
349+
if err != nil {
350+
return "", err
351+
}
352+
if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {
345353
return string(bytes), nil
346354
}
347355

@@ -457,16 +465,21 @@ func getAPI(env cliconfig.Environment, apiName string) (string, error) {
457465
return "", err
458466
}
459467

468+
var bytes []byte
460469
if _flagOutput == flags.JSONOutputType {
461-
bytes, err := libjson.Marshal(apisRes)
462-
if err != nil {
463-
return "", err
464-
}
470+
bytes, err = libjson.Marshal(apisRes)
471+
} else if _flagOutput == flags.YAMLOutputType {
472+
bytes, err = yaml.Marshal(apisRes)
473+
}
474+
if err != nil {
475+
return "", err
476+
}
477+
if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {
465478
return string(bytes), nil
466479
}
467480

468481
if len(apisRes) == 0 {
469-
exit.Error(errors.ErrorUnexpected(fmt.Sprintf("unable to find API %s", apiName)))
482+
exit.Error(errors.ErrorUnexpected(fmt.Sprintf("unable to find api %s", apiName)))
470483
}
471484

472485
apiRes := apisRes[0]

‎cli/cmd/lib_apis.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
Copyright 2021 Cortex Labs, Inc.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package cmd
18+
19+
import (
20+
"github.com/cortexlabs/cortex/pkg/lib/table"
21+
"github.com/cortexlabs/cortex/pkg/types/status"
22+
)
23+
24+
func replicaCountTable(counts *status.ReplicaCounts) table.Table {
25+
var rows [][]interface{}
26+
for _, replicaCountType := range status.ReplicaCountTypes {
27+
count := counts.GetCountBy(replicaCountType)
28+
canBeHiddenIfZero := false
29+
switch replicaCountType {
30+
case status.ReplicaCountFailed:
31+
canBeHiddenIfZero = true
32+
case status.ReplicaCountKilled:
33+
canBeHiddenIfZero = true
34+
case status.ReplicaCountKilledOOM:
35+
canBeHiddenIfZero = true
36+
case status.ReplicaCountErrImagePull:
37+
canBeHiddenIfZero = true
38+
case status.ReplicaCountUnknown:
39+
canBeHiddenIfZero = true
40+
case status.ReplicaCountStalled:
41+
canBeHiddenIfZero = true
42+
}
43+
if count == 0 && canBeHiddenIfZero {
44+
continue
45+
}
46+
rows = append(rows, []interface{}{
47+
replicaCountType,
48+
count,
49+
})
50+
}
51+
52+
return table.Table{
53+
Headers: []table.Header{
54+
{Title: _titleReplicaStatus, MinWidth: 32, MaxWidth: 32},
55+
{Title: _titleReplicaCount},
56+
},
57+
Rows: rows,
58+
}
59+
}

0 commit comments

Comments
 (0)