Skip to content

Commit 71da13f

Browse files
ewoutpewoutp
committed
Updated to master, fixed test
1 parent 3529bb6 commit 71da13f

File tree

1 file changed

+107
-64
lines changed

1 file changed

+107
-64
lines changed

‎tests/resilience_test.go‎

Lines changed: 107 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,19 @@ package tests
33
import (
44
"context"
55
"fmt"
6-
"os"
76
"testing"
87
"time"
98

10-
"github.com/dchest/uniuri"
9+
"github.com/dchest/uniuri"
1110
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12-
11+
1312
driver "github.com/arangodb/go-driver"
1413
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
1514
"github.com/arangodb/kube-arangodb/pkg/client"
16-
15+
"github.com/arangodb/kube-arangodb/pkg/util/retry"
1716
)
1817

19-
// TestResiliencePod
18+
// TestResiliencePod
2019
// Tests handling of individual pod deletions
2120
func TestResiliencePod(t *testing.T) {
2221
longOrSkip(t)
@@ -28,7 +27,7 @@ func TestResiliencePod(t *testing.T) {
2827

2928
// Prepare deployment config
3029
depl := newDeployment("test-pod-resilience" + uniuri.NewLen(4))
31-
depl.Spec.Mode = api.DeploymentModeCluster
30+
depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster)
3231
depl.Spec.SetDefaults(depl.GetName()) // this must be last
3332

3433
// Create deployment
@@ -38,7 +37,8 @@ func TestResiliencePod(t *testing.T) {
3837
}
3938

4039
// Wait for deployment to be ready
41-
if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentHasState(api.DeploymentStateRunning)); err != nil {
40+
apiObject, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentHasState(api.DeploymentStateRunning))
41+
if err != nil {
4242
t.Fatalf("Deployment not running in time: %v", err)
4343
}
4444

@@ -53,26 +53,40 @@ func TestResiliencePod(t *testing.T) {
5353
t.Fatalf("Cluster not running in expected health in time: %v", err)
5454
}
5555

56-
// Delete one pod after the other
57-
pods, err := kubecli.CoreV1().Pods(ns).List(metav1.ListOptions{})
58-
if err != nil {
59-
t.Fatalf("Could not find any pods in the %s, namespace: %v\n", ns, err)
60-
}
61-
fmt.Fprintf(os.Stderr,
62-
"There are %d pods in the %s namespace\n", len(pods.Items), ns)
63-
for _, pod := range pods.Items {
64-
if pod.GetName() == "arangodb-operator-test" { continue }
65-
fmt.Fprintf(os.Stderr,
66-
"Deleting pod %s in the %s namespace\n", pod.GetName(), ns)
67-
kubecli.CoreV1().Pods(ns).Delete(pod.GetName(),&metav1.DeleteOptions{})
68-
time.Sleep(30 * time.Second) // wait for problem to arise
69-
// Wait for cluster to be completely ready
70-
if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error {
71-
return clusterHealthEqualsSpec(h, apiObject.Spec)
72-
}); err != nil {
73-
t.Fatalf("Cluster not running in expected health in time: %v", err)
56+
// Delete one pod after the other
57+
apiObject.ForeachServerGroup(func(group api.ServerGroup, spec api.ServerGroupSpec, status *api.MemberStatusList) error {
58+
for _, m := range *status {
59+
// Get current pod so we can compare UID later
60+
originalPod, err := kubecli.CoreV1().Pods(ns).Get(m.PodName, metav1.GetOptions{})
61+
if err != nil {
62+
t.Fatalf("Failed to get pod %s: %v", m.PodName, err)
63+
}
64+
if err := kubecli.CoreV1().Pods(ns).Delete(m.PodName, &metav1.DeleteOptions{}); err != nil {
65+
t.Fatalf("Failed to delete pod %s: %v", m.PodName, err)
66+
}
67+
// Wait for pod to return with different UID
68+
op := func() error {
69+
pod, err := kubecli.CoreV1().Pods(ns).Get(m.PodName, metav1.GetOptions{})
70+
if err != nil {
71+
return maskAny(err)
72+
}
73+
if pod.GetUID() == originalPod.GetUID() {
74+
return fmt.Errorf("Still original pod")
75+
}
76+
return nil
77+
}
78+
if err := retry.Retry(op, time.Minute); err != nil {
79+
t.Fatalf("Pod did not restart: %v", err)
80+
}
81+
// Wait for cluster to be completely ready
82+
if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error {
83+
return clusterHealthEqualsSpec(h, apiObject.Spec)
84+
}); err != nil {
85+
t.Fatalf("Cluster not running in expected health in time: %v", err)
86+
}
7487
}
75-
}
88+
return nil
89+
}, &apiObject.Status)
7690

7791
// Cleanup
7892
removeDeployment(c, depl.GetName(), ns)
@@ -87,8 +101,8 @@ func TestResiliencePVC(t *testing.T) {
87101
ns := getNamespace(t)
88102

89103
// Prepare deployment config
90-
depl := newDeployment("test-pod-resilience" + uniuri.NewLen(4))
91-
depl.Spec.Mode = api.DeploymentModeCluster
104+
depl := newDeployment("test-pvc-resilience" + uniuri.NewLen(4))
105+
depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster)
92106
depl.Spec.SetDefaults(depl.GetName()) // this must be last
93107

94108
// Create deployment
@@ -98,7 +112,8 @@ func TestResiliencePVC(t *testing.T) {
98112
}
99113

100114
// Wait for deployment to be ready
101-
if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentHasState(api.DeploymentStateRunning)); err != nil {
115+
apiObject, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentHasState(api.DeploymentStateRunning))
116+
if err != nil {
102117
t.Fatalf("Deployment not running in time: %v", err)
103118
}
104119

@@ -113,30 +128,45 @@ func TestResiliencePVC(t *testing.T) {
113128
t.Fatalf("Cluster not running in expected health in time: %v", err)
114129
}
115130

116-
// Delete one pod after the other
117-
pvcs, err := kubecli.CoreV1().PersistentVolumeClaims(ns).List(metav1.ListOptions{})
118-
if err != nil {
119-
t.Fatalf("Could not find any persisted volume claims in the %s, namespace: %v\n", ns, err)
120-
}
121-
fmt.Fprintf(os.Stderr,
122-
"There are %d peristent volume claims in the %s namespace\n", len(pvcs.Items), ns)
123-
for _, pvc := range pvcs.Items {
124-
if pvc.GetName() == "arangodb-operator-test" { continue }
125-
fmt.Fprintf(os.Stderr,
126-
"Deleting persistent volume claim %s in the %s namespace\n", pvc.GetName(), ns)
127-
kubecli.CoreV1().PersistentVolumeClaims(ns).Delete(pvc.GetName(),&metav1.DeleteOptions{})
128-
time.Sleep(30 * time.Second) // wait for problem to arise
129-
// Wait for cluster to be completely ready
130-
if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error {
131-
return clusterHealthEqualsSpec(h, apiObject.Spec)
132-
}); err != nil {
133-
t.Fatalf("Cluster not running in expected health in time: %v", err)
131+
// Delete one pvc after the other
132+
apiObject.ForeachServerGroup(func(group api.ServerGroup, spec api.ServerGroupSpec, status *api.MemberStatusList) error {
133+
for _, m := range *status {
134+
// Get current pvc so we can compare UID later
135+
originalPVC, err := kubecli.CoreV1().PersistentVolumeClaims(ns).Get(m.PersistentVolumeClaimName, metav1.GetOptions{})
136+
if err != nil {
137+
t.Fatalf("Failed to get pvc %s: %v", m.PersistentVolumeClaimName, err)
138+
}
139+
if err := kubecli.CoreV1().PersistentVolumeClaims(ns).Delete(m.PersistentVolumeClaimName, &metav1.DeleteOptions{}); err != nil {
140+
t.Fatalf("Failed to delete pvc %s: %v", m.PersistentVolumeClaimName, err)
141+
}
142+
// Wait for pvc to return with different UID
143+
op := func() error {
144+
pvc, err := kubecli.CoreV1().PersistentVolumeClaims(ns).Get(m.PersistentVolumeClaimName, metav1.GetOptions{})
145+
if err != nil {
146+
return maskAny(err)
147+
}
148+
if pvc.GetUID() == originalPVC.GetUID() {
149+
return fmt.Errorf("Still original pvc")
150+
}
151+
return nil
152+
}
153+
if err := retry.Retry(op, time.Minute); err != nil {
154+
t.Fatalf("PVC did not restart: %v", err)
155+
}
156+
// Wait for cluster to be completely ready
157+
if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error {
158+
return clusterHealthEqualsSpec(h, apiObject.Spec)
159+
}); err != nil {
160+
t.Fatalf("Cluster not running in expected health in time: %v", err)
161+
}
134162
}
135-
}
163+
return nil
164+
}, &apiObject.Status)
136165

137166
// Cleanup
138167
removeDeployment(c, depl.GetName(), ns)
139168
}
169+
140170
// TestResilienceService
141171
// Tests handling of individual service deletions
142172
func TestResilienceService(t *testing.T) {
@@ -147,7 +177,7 @@ func TestResilienceService(t *testing.T) {
147177

148178
// Prepare deployment config
149179
depl := newDeployment("test-service-resilience" + uniuri.NewLen(4))
150-
depl.Spec.Mode = api.DeploymentModeCluster
180+
depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster)
151181
depl.Spec.SetDefaults(depl.GetName()) // this must be last
152182

153183
// Create deployment
@@ -157,8 +187,8 @@ func TestResilienceService(t *testing.T) {
157187
}
158188

159189
// Wait for deployment to be ready
160-
if _, err := waitUntilDeployment(
161-
c, depl.GetName(), ns, deploymentHasState(api.DeploymentStateRunning)); err != nil {
190+
apiObject, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentHasState(api.DeploymentStateRunning))
191+
if err != nil {
162192
t.Fatalf("Deployment not running in time: %v", err)
163193
}
164194

@@ -173,24 +203,37 @@ func TestResilienceService(t *testing.T) {
173203
t.Fatalf("Cluster not running in expected health in time: %v", err)
174204
}
175205

176-
// Delete one pod after the other
177-
services, err := kubecli.CoreV1().Services(ns).List(metav1.ListOptions{})
206+
// Delete database service
207+
// Get current pod so we can compare UID later
208+
serviceName := apiObject.Status.ServiceName
209+
originalService, err := kubecli.CoreV1().Services(ns).Get(serviceName, metav1.GetOptions{})
178210
if err != nil {
179-
t.Fatalf("Could not find any services in the %s, namespace: %v\n", ns, err)
211+
t.Fatalf("Failed to get service %s: %v", serviceName, err)
212+
}
213+
if err := kubecli.CoreV1().Services(ns).Delete(serviceName, &metav1.DeleteOptions{}); err != nil {
214+
t.Fatalf("Failed to delete service %s: %v", serviceName, err)
180215
}
181-
fmt.Fprintf(os.Stderr, "There are %d services in the %s namespace \n", len(services.Items), ns)
182-
for _, service := range services.Items {
183-
kubecli.CoreV1().Services(ns).Delete(service.GetName(),&metav1.DeleteOptions{})
184-
time.Sleep(30 * time.Second)
185-
// Wait for cluster to be completely ready
186-
if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error {
187-
return clusterHealthEqualsSpec(h, apiObject.Spec)
188-
}); err != nil {
189-
t.Fatalf("Cluster not running in expected health in time: %v", err)
216+
// Wait for service to return with different UID
217+
op := func() error {
218+
service, err := kubecli.CoreV1().Services(ns).Get(serviceName, metav1.GetOptions{})
219+
if err != nil {
220+
return maskAny(err)
221+
}
222+
if service.GetUID() == originalService.GetUID() {
223+
return fmt.Errorf("Still original service")
190224
}
225+
return nil
226+
}
227+
if err := retry.Retry(op, time.Minute); err != nil {
228+
t.Fatalf("PVC did not restart: %v", err)
229+
}
230+
// Wait for cluster to be completely ready
231+
if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error {
232+
return clusterHealthEqualsSpec(h, apiObject.Spec)
233+
}); err != nil {
234+
t.Fatalf("Cluster not running in expected health in time: %v", err)
191235
}
192236

193237
// Cleanup
194238
removeDeployment(c, depl.GetName(), ns)
195239
}
196-

0 commit comments

Comments
 (0)