Skip to content

Commit 4c1e178

Browse files
authored
Merge pull request #96 from arangodb/chaos
Adding chaos-monkey for deployments
2 parents 2437300 + bed3cb3 commit 4c1e178

File tree

15 files changed

+389
-5
lines changed

15 files changed

+389
-5
lines changed

Makefile

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,10 @@ ifndef ENTERPRISEIMAGE
6565
ENTERPRISEIMAGE := $(DEFAULTENTERPRISEIMAGE)
6666
endif
6767

68+
ifndef ALLOWCHAOS
69+
ALLOWCHAOS := true
70+
endif
71+
6872
BINNAME := $(PROJECT)
6973
BIN := $(BINDIR)/$(BINNAME)
7074
TESTBINNAME := $(PROJECT)_test
@@ -200,7 +204,8 @@ manifests: $(GOBUILDDIR)
200204
--output-suffix=$(MANIFESTSUFFIX) \
201205
--image=$(OPERATORIMAGE) \
202206
--image-sha256=$(IMAGESHA256) \
203-
--namespace=$(DEPLOYMENTNAMESPACE)
207+
--namespace=$(DEPLOYMENTNAMESPACE) \
208+
--allow-chaos=$(ALLOWCHAOS)
204209

205210
# Testing
206211

main.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,9 @@ var (
7979
operatorOptions struct {
8080
enableDeployment bool // Run deployment operator
8181
enableStorage bool // Run deployment operator
82-
createCRD bool
82+
}
83+
chaosOptions struct {
84+
allowed bool
8385
}
8486
deploymentProbe probe.Probe
8587
storageProbe probe.Probe
@@ -92,7 +94,7 @@ func init() {
9294
f.StringVar(&logLevel, "log.level", defaultLogLevel, "Set initial log level")
9395
f.BoolVar(&operatorOptions.enableDeployment, "operator.deployment", false, "Enable to run the ArangoDeployment operator")
9496
f.BoolVar(&operatorOptions.enableStorage, "operator.storage", false, "Enable to run the ArangoLocalStorage operator")
95-
f.BoolVar(&operatorOptions.createCRD, "operator.create-crd", true, "Disable to avoid create the custom resource definition")
97+
f.BoolVar(&chaosOptions.allowed, "chaos.allowed", false, "Set to allow chaos in deployments. Only activated when allowed and enabled in deployment")
9698
}
9799

98100
func main() {
@@ -188,7 +190,7 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
188190
ServiceAccount: serviceAccount,
189191
EnableDeployment: operatorOptions.enableDeployment,
190192
EnableStorage: operatorOptions.enableStorage,
191-
CreateCRD: operatorOptions.createCRD,
193+
AllowChaos: chaosOptions.allowed,
192194
}
193195
deps := operator.Dependencies{
194196
LogService: logService,

manifests/templates/deployment/deployment.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ spec:
1919
image: {{ .Image }}
2020
args:
2121
- --operator.deployment
22+
- --chaos.allowed={{ .Deployment.AllowChaos }}
2223
env:
2324
- name: MY_POD_NAMESPACE
2425
valueFrom:
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package v1alpha
24+
25+
import (
26+
time "time"
27+
28+
"github.com/arangodb/kube-arangodb/pkg/util"
29+
"github.com/pkg/errors"
30+
)
31+
32+
// ChaosSpec holds configuration for the deployment chaos monkey.
33+
type ChaosSpec struct {
34+
// Enabled switches the chaos monkey for a deployment on or off.
35+
Enabled *bool `json:"enabled,omitempty"`
36+
// Interval is the time between events
37+
Interval *time.Duration `json:"interval,omitempty"`
38+
// KillPodProbability is the chance of a pod being killed during an event
39+
KillPodProbability *Percent `json:"kill-pod-probability,omitempty"`
40+
}
41+
42+
// IsEnabled returns the value of enabled.
43+
func (s ChaosSpec) IsEnabled() bool {
44+
return util.BoolOrDefault(s.Enabled)
45+
}
46+
47+
// GetInterval returns the value of interval.
48+
func (s ChaosSpec) GetInterval() time.Duration {
49+
return util.DurationOrDefault(s.Interval)
50+
}
51+
52+
// GetKillPodProbability returns the value of kill-pod-probability.
53+
func (s ChaosSpec) GetKillPodProbability() Percent {
54+
return PercentOrDefault(s.KillPodProbability)
55+
}
56+
57+
// Validate the given spec
58+
func (s ChaosSpec) Validate() error {
59+
if s.IsEnabled() {
60+
if s.GetInterval() <= 0 {
61+
return maskAny(errors.Wrapf(ValidationError, "Interval must be > 0"))
62+
}
63+
if err := s.GetKillPodProbability().Validate(); err != nil {
64+
return maskAny(err)
65+
}
66+
}
67+
return nil
68+
}
69+
70+
// SetDefaults fills in missing defaults
71+
func (s *ChaosSpec) SetDefaults() {
72+
if s.GetInterval() == 0 {
73+
s.Interval = util.NewDuration(time.Minute)
74+
}
75+
if s.GetKillPodProbability() == 0 {
76+
s.KillPodProbability = NewPercent(50)
77+
}
78+
}
79+
80+
// SetDefaultsFrom fills unspecified fields with a value from given source spec.
81+
func (s *ChaosSpec) SetDefaultsFrom(source ChaosSpec) {
82+
if s.Enabled == nil {
83+
s.Enabled = util.NewBoolOrNil(source.Enabled)
84+
}
85+
if s.Interval == nil {
86+
s.Interval = util.NewDurationOrNil(source.Interval)
87+
}
88+
if s.KillPodProbability == nil {
89+
s.KillPodProbability = NewPercentOrNil(source.KillPodProbability)
90+
}
91+
}

pkg/apis/deployment/v1alpha/deployment_spec.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ type DeploymentSpec struct {
6262
Coordinators ServerGroupSpec `json:"coordinators"`
6363
SyncMasters ServerGroupSpec `json:"syncmasters"`
6464
SyncWorkers ServerGroupSpec `json:"syncworkers"`
65+
66+
Chaos ChaosSpec `json:"chaos"`
6567
}
6668

6769
// GetMode returns the value of mode.
@@ -147,6 +149,7 @@ func (s *DeploymentSpec) SetDefaults(deploymentName string) {
147149
s.Coordinators.SetDefaults(ServerGroupCoordinators, s.GetMode().HasCoordinators(), s.GetMode())
148150
s.SyncMasters.SetDefaults(ServerGroupSyncMasters, s.Sync.IsEnabled(), s.GetMode())
149151
s.SyncWorkers.SetDefaults(ServerGroupSyncWorkers, s.Sync.IsEnabled(), s.GetMode())
152+
s.Chaos.SetDefaults()
150153
}
151154

152155
// SetDefaultsFrom fills unspecified fields with a value from given source spec.
@@ -176,6 +179,7 @@ func (s *DeploymentSpec) SetDefaultsFrom(source DeploymentSpec) {
176179
s.Coordinators.SetDefaultsFrom(source.Coordinators)
177180
s.SyncMasters.SetDefaultsFrom(source.SyncMasters)
178181
s.SyncWorkers.SetDefaultsFrom(source.SyncWorkers)
182+
s.Chaos.SetDefaultsFrom(source.Chaos)
179183
}
180184

181185
// Validate the specification.
@@ -226,6 +230,9 @@ func (s *DeploymentSpec) Validate() error {
226230
if err := s.SyncWorkers.Validate(ServerGroupSyncWorkers, s.Sync.IsEnabled(), s.GetMode(), s.GetEnvironment()); err != nil {
227231
return maskAny(err)
228232
}
233+
if err := s.Chaos.Validate(); err != nil {
234+
return maskAny(errors.Wrap(err, "spec.chaos"))
235+
}
229236
return nil
230237
}
231238

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package v1alpha
24+
25+
import (
26+
"github.com/pkg/errors"
27+
)
28+
29+
// Percent is a percentage between 0 and 100.
30+
type Percent int
31+
32+
// Validate the given percentage.
33+
func (p Percent) Validate() error {
34+
if p < 0 || p > 100 {
35+
return maskAny(errors.Wrapf(ValidationError, "Percentage must be between 0 and 100, got %d", int(p)))
36+
}
37+
return nil
38+
}
39+
40+
// NewPercent returns a reference to a percent with given value.
41+
func NewPercent(input Percent) *Percent {
42+
return &input
43+
}
44+
45+
// NewPercentOrNil returns nil if input is nil, otherwise returns a clone of the given value.
46+
func NewPercentOrNil(input *Percent) *Percent {
47+
if input == nil {
48+
return nil
49+
}
50+
return NewPercent(*input)
51+
}
52+
53+
// PercentOrDefault returns the default value or 0 if input is nil, otherwise returns the referenced value.
54+
func PercentOrDefault(input *Percent, defaultValue ...Percent) Percent {
55+
if input == nil {
56+
if len(defaultValue) > 0 {
57+
return defaultValue[0]
58+
}
59+
return 0
60+
}
61+
return *input
62+
}

pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,49 @@ func (in *AuthenticationSpec) DeepCopy() *AuthenticationSpec {
144144
return out
145145
}
146146

147+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
148+
func (in *ChaosSpec) DeepCopyInto(out *ChaosSpec) {
149+
*out = *in
150+
if in.Enabled != nil {
151+
in, out := &in.Enabled, &out.Enabled
152+
if *in == nil {
153+
*out = nil
154+
} else {
155+
*out = new(bool)
156+
**out = **in
157+
}
158+
}
159+
if in.Interval != nil {
160+
in, out := &in.Interval, &out.Interval
161+
if *in == nil {
162+
*out = nil
163+
} else {
164+
*out = new(time.Duration)
165+
**out = **in
166+
}
167+
}
168+
if in.KillPodProbability != nil {
169+
in, out := &in.KillPodProbability, &out.KillPodProbability
170+
if *in == nil {
171+
*out = nil
172+
} else {
173+
*out = new(Percent)
174+
**out = **in
175+
}
176+
}
177+
return
178+
}
179+
180+
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChaosSpec.
181+
func (in *ChaosSpec) DeepCopy() *ChaosSpec {
182+
if in == nil {
183+
return nil
184+
}
185+
out := new(ChaosSpec)
186+
in.DeepCopyInto(out)
187+
return out
188+
}
189+
147190
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
148191
func (in *Condition) DeepCopyInto(out *Condition) {
149192
*out = *in
@@ -220,6 +263,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
220263
in.Coordinators.DeepCopyInto(&out.Coordinators)
221264
in.SyncMasters.DeepCopyInto(&out.SyncMasters)
222265
in.SyncWorkers.DeepCopyInto(&out.SyncWorkers)
266+
in.Chaos.DeepCopyInto(&out.Chaos)
223267
return
224268
}
225269

pkg/deployment/chaos/context.go

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package chaos
24+
25+
import (
26+
"k8s.io/api/core/v1"
27+
28+
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
29+
)
30+
31+
// Context provides methods to the chaos package.
32+
type Context interface {
33+
// GetSpec returns the current specification of the deployment
34+
GetSpec() api.DeploymentSpec
35+
// DeletePod deletes a pod with given name in the namespace
36+
// of the deployment. If the pod does not exist, the error is ignored.
37+
DeletePod(podName string) error
38+
// GetOwnedPods returns a list of all pods owned by the deployment.
39+
GetOwnedPods() ([]v1.Pod, error)
40+
}

pkg/deployment/chaos/errors.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package chaos
24+
25+
import "github.com/pkg/errors"
26+
27+
var (
28+
maskAny = errors.WithStack
29+
)

0 commit comments

Comments
 (0)