Skip to content

Commit d9aa13c

Browse files
informalictinformalict
authored
[Feature] Check if the server is cleaned out (#775)
1 parent f7e5453 commit d9aa13c

File tree

8 files changed

+216
-7
lines changed

8 files changed

+216
-7
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
- Add HighPriorityPlan to ArangoDeployment Status
66
- Add Pending Member phase
77
- Add Ephemeral Volumes for apps feature
8+
- Check if the DB server is cleaned out.
89

910
## [1.2.1](https://github.com/arangodb/kube-arangodb/tree/1.2.1) (2021-07-28)
1011
- Fix ArangoMember race with multiple ArangoDeployments within single namespace

‎pkg/apis/deployment/v1/plan.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func (a ActionType) String() string {
4949
// Priority returns plan priority
5050
func (a ActionType) Priority() ActionPriority {
5151
switch a {
52-
case ActionTypeMemberPhaseUpdate, ActionTypeMemberRIDUpdate:
52+
case ActionTypeMemberPhaseUpdate, ActionTypeMemberRIDUpdate, ActionTypeSetMemberCondition:
5353
return ActionPriorityHigh
5454
default:
5555
return ActionPriorityNormal
@@ -147,7 +147,7 @@ const (
147147
ActionTypeClusterMemberCleanup ActionType = "ClusterMemberCleanup"
148148
// ActionTypeEnableMaintenance enables maintenance on cluster.
149149
ActionTypeEnableMaintenance ActionType = "EnableMaintenance"
150-
// ActionTypeEnableMaintenance disables maintenance on cluster.
150+
// ActionTypeDisableMaintenance disables maintenance on cluster.
151151
ActionTypeDisableMaintenance ActionType = "DisableMaintenance"
152152
// ActionTypeSetMaintenanceCondition sets maintenance condition.
153153
ActionTypeSetMaintenanceCondition ActionType = "SetMaintenanceCondition"
@@ -157,6 +157,8 @@ const (
157157
ActionTypeBootstrapSetPassword ActionType = "BootstrapSetPassword"
158158
// ActionTypeMemberPhaseUpdate updated member phase. High priority
159159
ActionTypeMemberPhaseUpdate ActionType = "MemberPhaseUpdate"
160+
// ActionTypeSetMemberCondition sets member condition. It is high priority action.
161+
ActionTypeSetMemberCondition ActionType = "SetMemberCondition"
160162
// ActionTypeMemberRIDUpdate updated member Run ID (UID). High priority
161163
ActionTypeMemberRIDUpdate ActionType = "MemberRIDUpdate"
162164
)

‎pkg/apis/deployment/v2alpha1/plan.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func (a ActionType) String() string {
4949
// Priority returns plan priority
5050
func (a ActionType) Priority() ActionPriority {
5151
switch a {
52-
case ActionTypeMemberPhaseUpdate, ActionTypeMemberRIDUpdate:
52+
case ActionTypeMemberPhaseUpdate, ActionTypeMemberRIDUpdate, ActionTypeSetMemberCondition:
5353
return ActionPriorityHigh
5454
default:
5555
return ActionPriorityNormal
@@ -147,7 +147,7 @@ const (
147147
ActionTypeClusterMemberCleanup ActionType = "ClusterMemberCleanup"
148148
// ActionTypeEnableMaintenance enables maintenance on cluster.
149149
ActionTypeEnableMaintenance ActionType = "EnableMaintenance"
150-
// ActionTypeEnableMaintenance disables maintenance on cluster.
150+
// ActionTypeDisableMaintenance disables maintenance on cluster.
151151
ActionTypeDisableMaintenance ActionType = "DisableMaintenance"
152152
// ActionTypeSetMaintenanceCondition sets maintenance condition.
153153
ActionTypeSetMaintenanceCondition ActionType = "SetMaintenanceCondition"
@@ -157,6 +157,8 @@ const (
157157
ActionTypeBootstrapSetPassword ActionType = "BootstrapSetPassword"
158158
// ActionTypeMemberPhaseUpdate updated member phase. High priority
159159
ActionTypeMemberPhaseUpdate ActionType = "MemberPhaseUpdate"
160+
// ActionTypeSetMemberCondition sets member condition. It is high priority action.
161+
ActionTypeSetMemberCondition ActionType = "SetMemberCondition"
160162
// ActionTypeMemberRIDUpdate updated member Run ID (UID). High priority
161163
ActionTypeMemberRIDUpdate ActionType = "MemberRIDUpdate"
162164
)

‎pkg/deployment/reconcile/action_cleanout_member.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e
178178
}
179179
// Cleanout completed
180180
if m.Conditions.Update(api.ConditionTypeCleanedOut, true, "CleanedOut", "") {
181-
if a.actionCtx.UpdateMember(ctx, m); err != nil {
181+
if err := a.actionCtx.UpdateMember(ctx, m); err != nil {
182182
return false, false, errors.WithStack(err)
183183
}
184184
}

‎pkg/deployment/reconcile/action_set_member_condition.go‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2021 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Tomasz Mielech
21+
//
22+
23+
package reconcile
24+
25+
import (
26+
"context"
27+
"strconv"
28+
29+
"github.com/rs/zerolog"
30+
31+
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
32+
"github.com/arangodb/kube-arangodb/pkg/util/errors"
33+
)
34+
35+
func init() {
36+
registerAction(api.ActionTypeSetMemberCondition, setMemberCondition)
37+
}
38+
39+
func setMemberCondition(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
40+
a := &actionSetMemberCondition{}
41+
42+
a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
43+
44+
return a
45+
}
46+
47+
type actionSetMemberCondition struct {
48+
// actionImpl implement timeout and member id functions
49+
actionImpl
50+
51+
actionEmptyCheckProgress
52+
}
53+
54+
// Start starts the action for changing conditions on the provided member.
55+
func (a actionSetMemberCondition) Start(ctx context.Context) (bool, error) {
56+
m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID)
57+
if !ok {
58+
a.log.Info().Msg("can not set the condition because the member is gone already")
59+
return true, nil
60+
}
61+
62+
if len(a.action.Params) == 0 {
63+
a.log.Info().Msg("can not start the action with the empty list of conditions")
64+
return true, nil
65+
}
66+
67+
for condition, value := range a.action.Params {
68+
set, err := strconv.ParseBool(value)
69+
if err != nil {
70+
a.log.Error().Err(err).Str("value", value).Msg("can not parse string to boolean")
71+
continue
72+
}
73+
74+
a.log.Debug().Msg("set the condition")
75+
76+
m.Conditions.Update(api.ConditionType(condition), set, a.action.Reason, "action set the member condition")
77+
}
78+
79+
if err := a.actionCtx.UpdateMember(ctx, m); err != nil {
80+
return false, errors.Wrap(errors.WithStack(err), "can not update the member")
81+
}
82+
83+
return true, nil
84+
}

‎pkg/deployment/reconcile/plan_builder_clean_out.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2021 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Tomasz Mielech
21+
//
22+
23+
package reconcile
24+
25+
import (
26+
"context"
27+
"strconv"
28+
29+
"github.com/arangodb/go-driver"
30+
"github.com/rs/zerolog"
31+
32+
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
33+
"github.com/arangodb/kube-arangodb/pkg/util/arangod"
34+
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
35+
inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
36+
)
37+
38+
// createCleanOutPlan creates clean out action if the server is cleaned out and the operator is not aware of it.
39+
func createCleanOutPlan(ctx context.Context, log zerolog.Logger, _ k8sutil.APIObject, spec api.DeploymentSpec,
40+
status api.DeploymentStatus, _ inspectorInterface.Inspector, planCtx PlanBuilderContext) api.Plan {
41+
42+
if spec.GetMode() != api.DeploymentModeCluster {
43+
return nil
44+
}
45+
46+
cluster, err := getCluster(ctx, planCtx)
47+
if err != nil {
48+
log.Warn().Err(err).Msgf("Unable to get cluster")
49+
return nil
50+
}
51+
52+
ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout())
53+
defer cancel()
54+
health, err := cluster.Health(ctxChild)
55+
if err != nil {
56+
log.Warn().Err(err).Msgf("Unable to get cluster health")
57+
return nil
58+
}
59+
60+
for id, member := range health.Health {
61+
switch member.Role {
62+
case driver.ServerRoleDBServer:
63+
memberStatus, ok := status.Members.DBServers.ElementByID(string(id))
64+
if !ok {
65+
continue
66+
}
67+
68+
if memberStatus.Conditions.IsTrue(api.ConditionTypeCleanedOut) {
69+
continue
70+
}
71+
72+
if isCleanedOut, err := cluster.IsCleanedOut(ctx, string(id)); err != nil {
73+
log.Warn().Err(err).Str("id", string(id)).Msgf("Unable to get clean out status")
74+
return nil
75+
} else if isCleanedOut {
76+
log.Info().
77+
Str("role", string(member.Role)).
78+
Str("id", string(id)).
79+
Msgf("server is cleaned out so operator must do the same")
80+
81+
action := api.NewAction(api.ActionTypeSetMemberCondition, api.ServerGroupDBServers, string(id),
82+
"server is cleaned out so operator must do the same")
83+
action = action.AddParam(string(api.ConditionTypeCleanedOut), strconv.FormatBool(true))
84+
85+
return api.Plan{action}
86+
}
87+
}
88+
}
89+
90+
return nil
91+
}

‎pkg/deployment/reconcile/plan_builder_high.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ func createHighPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.A
8686
plan = pb.Apply(updateMemberPhasePlan)
8787
}
8888

89+
if plan.IsEmpty() {
90+
plan = pb.Apply(createCleanOutPlan)
91+
}
92+
8993
// Return plan
9094
return plan, true
9195
}

‎pkg/deployment/reconcile/utils.go‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
//
22
// DISCLAIMER
33
//
4-
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
4+
// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany
55
//
66
// Licensed under the Apache License, Version 2.0 (the "License");
77
// you may not use this file except in compliance with the License.
@@ -18,15 +18,21 @@
1818
// Copyright holder is ArangoDB GmbH, Cologne, Germany
1919
//
2020
// Author Adam Janikowski
21+
// Author Tomasz Mielech
2122
//
2223

2324
package reconcile
2425

2526
import (
27+
"context"
2628
"sort"
2729

28-
"github.com/arangodb/kube-arangodb/pkg/util"
30+
"github.com/arangodb/go-driver"
2931
core "k8s.io/api/core/v1"
32+
33+
"github.com/arangodb/kube-arangodb/pkg/util"
34+
"github.com/arangodb/kube-arangodb/pkg/util/arangod"
35+
"github.com/arangodb/kube-arangodb/pkg/util/errors"
3036
)
3137

3238
func secretKeysToListWithPrefix(s *core.Secret) []string {
@@ -44,3 +50,22 @@ func secretKeysToList(s *core.Secret) []string {
4450

4551
return keys
4652
}
53+
54+
// getCluster returns the cluster connection.
55+
func getCluster(ctx context.Context, planCtx PlanBuilderContext) (driver.Cluster, error) {
56+
ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout())
57+
defer cancel()
58+
c, err := planCtx.GetDatabaseClient(ctxChild)
59+
if err != nil {
60+
return nil, errors.WithStack(errors.Wrapf(err, "Unable to get database client"))
61+
}
62+
63+
ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
64+
defer cancel()
65+
cluster, err := c.Cluster(ctxChild)
66+
if err != nil {
67+
return nil, errors.WithStack(errors.Wrapf(err, "Unable to get cluster client"))
68+
}
69+
70+
return cluster, nil
71+
}

0 commit comments

Comments
 (0)