K8SPS-73 - Add self-healing test

Author: tplavcic

e2e-tests/functions

@@ -547,16 +547,17 @@ kill_pods() {
 	local selector=$2
 	local pod_label=$3
 	local label_value=$4
+	local chaos_suffix=$5
 
 	if [ "${selector}" == "pod" ]; then
 		yq eval '
-			.metadata.name = "chaos-pod-kill-'${RANDOM}'" |
+			.metadata.name = "chaos-pod-kill-'${chaos_suffix}'" |
 			del(.spec.selector.pods.test-namespace) |
 			.spec.selector.pods.'${ns}'[0] = "'${pod_label}'"' ${TESTS_CONFIG_DIR}/chaos-pod-kill.yml \
 			| kubectl apply --namespace ${ns} -f -
 	elif [ "${selector}" == "label" ]; then
 		yq eval '
-			.metadata.name = "chaos-kill-label-'${RANDOM}'" |
+			.metadata.name = "chaos-kill-label-'${chaos_suffix}'" |
 			.spec.mode = "all" |
 			del(.spec.selector.pods) |
 			.spec.selector.labelSelectors."'${pod_label}'" = "'${label_value}'"' ${TESTS_CONFIG_DIR}/chaos-pod-kill.yml \
@@ -568,9 +569,10 @@ kill_pods() {
 failure_pod() {
 	local ns=$1
 	local pod=$2
+	local chaos_suffix=$3
 
 	yq eval '
-        .metadata.name = "chaos-pod-failure-'${RANDOM}'" |
+        .metadata.name = "chaos-pod-failure-'${chaos_suffix}'" |
         del(.spec.selector.pods.test-namespace) |
         .spec.selector.pods.'${ns}'[0] = "'${pod}'"' ${TESTS_CONFIG_DIR}/chaos-pod-failure.yml \
 		| kubectl apply --namespace ${ns} -f -
@@ -580,9 +582,10 @@ failure_pod() {
 network_loss() {
 	local ns=$1
 	local pod=$2
+	local chaos_suffix=$3
 
 	yq eval '
-        .metadata.name = "chaos-pod-network-loss-'${RANDOM}'" |
+        .metadata.name = "chaos-pod-network-loss-'${chaos_suffix}'" |
         del(.spec.selector.pods.test-namespace) |
         .spec.selector.pods.'${ns}'[0] = "'${pod}'"' ${TESTS_CONFIG_DIR}/chaos-network-loss.yml \
 		| kubectl apply --namespace ${ns} -f -

e2e-tests/run-distro.csv

@@ -14,6 +14,7 @@ monitoring
 one-pod
 operator-self-healing
 scaling
+self-healing
 service-per-pod
 sidecars
 tls-cert-manager

e2e-tests/run-minikube.csv

@@ -12,6 +12,7 @@ haproxy
 init-deploy
 one-pod
 operator-self-healing
+self-healing
 sidecars
 tls-cert-manager
 users

e2e-tests/run-pr.csv

@@ -18,6 +18,7 @@ monitoring
 one-pod
 operator-self-healing
 scaling
+self-healing
 service-per-pod
 sidecars
 tls-cert-manager

e2e-tests/run-release.csv

@@ -17,6 +17,7 @@ monitoring
 one-pod
 operator-self-healing
 scaling
+self-healing
 service-per-pod
 sidecars
 tls-cert-manager

e2e-tests/tests/self-healing/00-assert.yaml

@@ -0,0 +1,26 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: perconaservermysqls.ps.percona.com
+spec:
+  group: ps.percona.com
+  names:
+    kind: PerconaServerMySQL
+    listKind: PerconaServerMySQLList
+    plural: perconaservermysqls
+    shortNames:
+    - ps
+    singular: perconaservermysql
+  scope: Namespaced
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: check-operator-deploy-status
+timeout: 120
+commands:
+  - script: kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1

e2e-tests/tests/self-healing/00-deploy-operator.yaml

@@ -0,0 +1,14 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      deploy_operator
+      deploy_non_tls_cluster_secrets
+      deploy_tls_cluster_secrets
+      deploy_client

e2e-tests/tests/self-healing/01-assert.yaml

@@ -0,0 +1,27 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: chaos-controller-manager
+spec:
+  replicas: 3
+status:
+  availableReplicas: 3
+  readyReplicas: 3
+  replicas: 3
+  updatedReplicas: 3
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: chaos-daemon
+status:
+  currentNumberScheduled: 3
+  desiredNumberScheduled: 3
+  numberAvailable: 3
+  numberMisscheduled: 0
+  numberReady: 3
+  updatedNumberScheduled: 3

e2e-tests/tests/self-healing/01-deploy-chaos-mesh.yaml

@@ -0,0 +1,11 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      deploy_chaos_mesh

e2e-tests/tests/self-healing/02-assert.yaml

@@ -0,0 +1,60 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 420
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: self-healing-mysql
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: self-healing-orc
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: self-healing-haproxy
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+apiVersion: ps.percona.com/v1alpha1
+kind: PerconaServerMySQL
+metadata:
+  name: self-healing
+  finalizers:
+    - delete-mysql-pods-in-order
+status:
+  haproxy:
+    ready: 3
+    size: 3
+    state: ready
+  mysql:
+    ready: 3
+    size: 3
+    state: ready
+  orchestrator:
+    ready: 3
+    size: 3
+    state: ready
+  state: ready