Skip to content

Commit aafeff7

Browse files
authored
[Feature] Change rotation discovery (#531)
1 parent 221d9d1 commit aafeff7

File tree

12 files changed

+686
-269
lines changed

12 files changed

+686
-269
lines changed

go.mod

Lines changed: 15 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module github.com/arangodb/kube-arangodb
33
go 1.13
44

55
replace (
6+
github.com/coreos/prometheus-operator => github.com/coreos/prometheus-operator v0.37.0
67
github.com/ugorji/go => github.com/ugorji/go v0.0.0-20181209151446-772ced7fd4c2
78

89
k8s.io/api => k8s.io/api v0.15.9
@@ -13,53 +14,38 @@ replace (
1314
)
1415

1516
require (
16-
github.com/PuerkitoBio/purell v1.1.1 // indirect
17-
github.com/aktau/github-release v0.7.2 // indirect
1817
github.com/arangodb-helper/go-certificates v0.0.0-20180821055445-9fca24fc2680
1918
github.com/arangodb/arangosync-client v0.6.3
2019
github.com/arangodb/go-driver v0.0.0-20191002124627-11b6bfc64f67
2120
github.com/arangodb/go-upgrade-rules v0.0.0-20180809110947-031b4774ff21
2221
github.com/cenkalti/backoff v2.1.1+incompatible
2322
github.com/coreos/go-semver v0.3.0
24-
github.com/coreos/prometheus-operator v0.31.1
23+
github.com/coreos/prometheus-operator v0.37.0
2524
github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9
2625
github.com/dgrijalva/jwt-go v3.2.0+incompatible
27-
github.com/evanphx/json-patch v4.2.0+incompatible
26+
github.com/evanphx/json-patch v4.5.0+incompatible
2827
github.com/ghodss/yaml v1.0.0
2928
github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 // indirect
3029
github.com/gin-gonic/gin v1.3.0
31-
github.com/go-openapi/spec v0.18.0 // indirect
32-
github.com/go-openapi/swag v0.18.0 // indirect
33-
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
34-
github.com/google/gofuzz v1.0.0 // indirect
35-
github.com/google/uuid v1.1.1 // indirect
36-
github.com/googleapis/gnostic v0.2.0 // indirect
3730
github.com/jessevdk/go-assets v0.0.0-20160921144138-4f4301a06e15
3831
github.com/jessevdk/go-assets-builder v0.0.0-20130903091706-b8483521738f
39-
github.com/jessevdk/go-flags v1.4.0 // indirect
40-
github.com/julienschmidt/httprouter v1.2.0
32+
github.com/julienschmidt/httprouter v1.3.0
4133
github.com/magiconair/properties v1.8.0
42-
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe // indirect
43-
github.com/mattn/go-isatty v0.0.7 // indirect
34+
github.com/mattn/go-isatty v0.0.12 // indirect
4435
github.com/pkg/errors v0.8.1
45-
github.com/prometheus/client_golang v1.0.0
36+
github.com/prometheus/client_golang v1.2.1
4637
github.com/robfig/cron v1.2.0
4738
github.com/rs/zerolog v1.14.3
48-
github.com/spf13/cobra v0.0.3
49-
github.com/spf13/pflag v1.0.3
50-
github.com/stretchr/testify v1.3.0
51-
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
39+
github.com/spf13/cobra v0.0.5
40+
github.com/spf13/pflag v1.0.5
41+
github.com/stretchr/testify v1.4.0
5242
github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2 // indirect
53-
github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2 // indirect
54-
golang.org/x/sys v0.0.0-20190506115046-ca7f33d4116e
55-
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
43+
golang.org/x/sys v0.0.0-20200116001909-b77594299b42
5644
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
5745
gopkg.in/go-playground/validator.v8 v8.18.2 // indirect
58-
k8s.io/api v0.15.9
59-
k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8
60-
k8s.io/apimachinery v0.15.9
61-
k8s.io/client-go v11.0.0+incompatible
62-
k8s.io/klog v0.3.1
63-
k8s.io/kube-openapi v0.0.0-20190502190224-411b2483e503 // indirect
64-
k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5 // indirect
46+
k8s.io/api v0.17.3
47+
k8s.io/apiextensions-apiserver v0.17.3
48+
k8s.io/apimachinery v0.17.3
49+
k8s.io/client-go v12.0.0+incompatible
50+
k8s.io/klog v1.0.0
6551
)

go.sum

Lines changed: 381 additions & 136 deletions
Large diffs are not rendered by default.

pkg/apis/deployment/v1/member_status.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ import (
2626
"reflect"
2727
"time"
2828

29+
"k8s.io/apimachinery/pkg/types"
30+
2931
driver "github.com/arangodb/go-driver"
3032
"github.com/arangodb/kube-arangodb/pkg/util"
3133
v1 "k8s.io/api/core/v1"
@@ -45,6 +47,10 @@ type MemberStatus struct {
4547
PersistentVolumeClaimName string `json:"persistentVolumeClaimName,omitempty"`
4648
// PodName holds the name of the Pod that currently runs this member
4749
PodName string `json:"podName,omitempty"`
50+
// PodUID holds the UID of the Pod that currently runs this member
51+
PodUID types.UID `json:"podUID,omitempty"`
52+
// PodSpecVersion holds the checksum of Pod spec that currently runs this member. Used to rotate pods
53+
PodSpecVersion string `json:"podSpecVersion,omitempty"`
4854
// Conditions specific to this member
4955
Conditions ConditionList `json:"conditions,omitempty"`
5056
// RecentTerminatons holds the times when this member was recently terminated.

pkg/deployment/context_impl.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -467,3 +467,11 @@ func (d *Deployment) GetAgencyData(ctx context.Context, i interface{}, keyParts
467467

468468
return err
469469
}
470+
471+
func (d *Deployment) RenderPodForMember(spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*v1.Pod, error) {
472+
return d.resources.RenderPodForMember(spec, status, memberID, imageInfo)
473+
}
474+
475+
func (d *Deployment) SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) {
476+
return d.resources.SelectImage(spec, status)
477+
}

pkg/deployment/deployment_test.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
package deployment
2424

2525
import (
26+
"fmt"
2627
"io/ioutil"
2728
"os"
2829
"testing"
@@ -1885,7 +1886,10 @@ func TestEnsurePods(t *testing.T) {
18851886

18861887
// Assert
18871888
if testCase.ExpectedError != nil {
1888-
assert.EqualError(t, err, testCase.ExpectedError.Error())
1889+
1890+
if !assert.EqualError(t, err, testCase.ExpectedError.Error()) {
1891+
println(fmt.Sprintf("%+v", err))
1892+
}
18891893
return
18901894
}
18911895

pkg/deployment/images.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,13 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima
195195
image: image,
196196
}
197197

198-
if err := resources.CreateArangoPod(ib.KubeCli, ib.APIObject, role, id, podName, args, &imagePod); err != nil {
198+
pod, err := resources.RenderArangoPod(ib.APIObject, role, id, podName, args, &imagePod)
199+
if err != nil {
200+
log.Debug().Err(err).Msg("Failed to render image ID pod")
201+
return true, maskAny(err)
202+
}
203+
204+
if _, _, err := resources.CreateArangoPod(ib.KubeCli, ib.APIObject, pod); err != nil {
199205
log.Debug().Err(err).Msg("Failed to create image ID pod")
200206
return true, maskAny(err)
201207
}

pkg/deployment/reconcile/context.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,4 +112,8 @@ type Context interface {
112112
EnableScalingCluster() error
113113
// GetAgencyData object for key path
114114
GetAgencyData(ctx context.Context, i interface{}, keyParts ...string) error
115+
// Renders Pod definition for member
116+
RenderPodForMember(spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*v1.Pod, error)
117+
// SelectImage select currently used image by pod
118+
SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool)
115119
}

pkg/deployment/reconcile/plan_builder_context.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828
driver "github.com/arangodb/go-driver"
2929
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
3030
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
31-
v1 "k8s.io/api/core/v1"
31+
core "k8s.io/api/core/v1"
3232
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3333
)
3434

@@ -44,7 +44,7 @@ type PlanBuilderContext interface {
4444
// On error, the error is logged.
4545
CreateEvent(evt *k8sutil.Event)
4646
// GetPvc gets a PVC by the given name, in the samespace of the deployment.
47-
GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error)
47+
GetPvc(pvcName string) (*core.PersistentVolumeClaim, error)
4848
// GetExpectedPodArguments creates command line arguments for a server in the given group with given ID.
4949
GetExpectedPodArguments(apiObject metav1.Object, deplSpec api.DeploymentSpec, group api.ServerGroup,
5050
agents api.MemberStatusList, id string, version driver.Version) []string
@@ -56,6 +56,10 @@ type PlanBuilderContext interface {
5656
GetStatus() (api.DeploymentStatus, int32)
5757
// GetAgencyData object for key path
5858
GetAgencyData(ctx context.Context, i interface{}, keyParts ...string) error
59+
// Renders Pod definition for member
60+
RenderPodForMember(spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error)
61+
// SelectImage select currently used image by pod
62+
SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool)
5963
}
6064

6165
// newPlanBuilderContext creates a PlanBuilderContext from the given context

pkg/deployment/reconcile/plan_builder_rotate_upgrade.go

Lines changed: 51 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@ import (
2929
"github.com/arangodb/kube-arangodb/pkg/apis/deployment"
3030
"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
3131

32-
"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
33-
3432
"github.com/arangodb/go-driver"
3533
upgraderules "github.com/arangodb/go-upgrade-rules"
3634
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
@@ -85,9 +83,17 @@ func createRotateOrUpgradePlan(log zerolog.Logger, apiObject k8sutil.APIObject,
8583
!decision.AutoUpgradeNeeded)
8684
} else {
8785
// Upgrade is not needed, see if rotation is needed
88-
rotNeeded, reason := podNeedsRotation(log, pod, apiObject, spec, group, status, m.ID, context)
89-
if rotNeeded {
90-
newPlan = createRotateMemberPlan(log, m, group, reason)
86+
if m.PodUID == "" {
87+
rotNeeded, reason := podNeedsRotation(log, pod, apiObject, spec, group, status, m.ID, context)
88+
if rotNeeded {
89+
newPlan = createRotateMemberPlan(log, m, group, reason)
90+
}
91+
} else {
92+
// Use new level of rotate logic
93+
rotNeeded, reason := podNeedsRotationNew(log, pod, apiObject, spec, group, status, m, context)
94+
if rotNeeded {
95+
newPlan = createRotateMemberPlan(log, m, group, reason)
96+
}
9197
}
9298
}
9399

@@ -186,6 +192,46 @@ func podNeedsUpgrading(log zerolog.Logger, p core.Pod, spec api.DeploymentSpec,
186192
return upgradeDecision{UpgradeNeeded: false}
187193
}
188194

195+
// podNeedsRotationNew returns true when the specification of the
196+
// given pod differs from what it should be according to the
197+
// given deployment spec.
198+
// When true is returned, a reason for the rotation is already returned.
199+
func podNeedsRotationNew(log zerolog.Logger, p core.Pod, apiObject metav1.Object, spec api.DeploymentSpec,
200+
group api.ServerGroup, status api.DeploymentStatus, m api.MemberStatus,
201+
context PlanBuilderContext) (bool, string) {
202+
if m.PodUID != p.UID {
203+
return true, "Pod UID does not match, this pod is not managed by Operator. Recreating"
204+
}
205+
206+
if m.PodSpecVersion == "" {
207+
return true, "Pod Spec Version is nil - recreating pod"
208+
}
209+
210+
imageInfo, imageFound := context.SelectImage(spec, status)
211+
if !imageFound {
212+
// Image is not found, so rotation is not needed
213+
return false, ""
214+
}
215+
216+
renderedPod, err := context.RenderPodForMember(spec, status, m.ID, imageInfo)
217+
if err != nil {
218+
log.Err(err).Msg("Error while rendering pod")
219+
return false, ""
220+
}
221+
222+
checksum, err := k8sutil.GetPodSpecChecksum(renderedPod.Spec)
223+
if err != nil {
224+
log.Err(err).Msg("Error while getting pod checksum")
225+
return false, ""
226+
}
227+
228+
if m.PodSpecVersion != checksum {
229+
return true, "Pod needs rotation - checksum does not match"
230+
}
231+
232+
return false, ""
233+
}
234+
189235
// podNeedsRotation returns true when the specification of the
190236
// given pod differs from what it should be according to the
191237
// given deployment spec.

pkg/deployment/reconcile/plan_builder_test.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,14 @@ type testContext struct {
5454
RecordedEvent *k8sutil.Event
5555
}
5656

57+
func (c *testContext) RenderPodForMember(spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) {
58+
panic("implement me")
59+
}
60+
61+
func (c *testContext) SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) {
62+
panic("implement me")
63+
}
64+
5765
func (c *testContext) UpdatePvc(pvc *core.PersistentVolumeClaim) error {
5866
panic("implement me")
5967
}

0 commit comments

Comments
 (0)