K8SPS-73 - Add gr-self-healing test

Author: tplavcic

Jenkinsfile

@@ -275,6 +275,7 @@ void prepareNode() {
         # v0.15.0 kuttl version
         kubectl krew install --manifest-url https://raw.githubusercontent.com/kubernetes-sigs/krew-index/a67f31ecb2e62f15149ca66d096357050f07b77d/plugins/kuttl.yaml
         printf "%s is installed" "$(kubectl kuttl --version)"
+        kubectl krew install assert
     '''
 }
 

e2e-tests/functions

@@ -476,6 +476,10 @@ get_primary_from_haproxy() {
 	run_mysql "SHOW VARIABLES LIKE '%hostname%';" "-h ${haproxy_pod_ip} -P3306 -uroot -proot_password" | awk '{print $2}'
 }
 
+get_primary_from_group_replication() {
+	run_mysql "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='PRIMARY';" "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" | cut -d'.' -f1
+}
+
 verify_certificate_sans() {
 	local certificate=$1
 	local expected_sans=$2

e2e-tests/run-distro.csv

@@ -7,6 +7,7 @@ gr-haproxy
 gr-init-deploy
 gr-one-pod
 gr-scaling
+gr-self-healing
 gr-tls-cert-manager
 haproxy
 init-deploy

e2e-tests/run-minikube.csv

@@ -7,6 +7,7 @@ gr-haproxy
 gr-init-deploy
 gr-one-pod
 gr-scaling
+gr-self-healing
 gr-tls-cert-manager
 haproxy
 init-deploy

e2e-tests/run-pr.csv

@@ -10,6 +10,7 @@ gr-ignore-annotations
 gr-init-deploy
 gr-one-pod
 gr-scaling
+gr-self-healing
 gr-tls-cert-manager
 haproxy
 init-deploy

e2e-tests/run-release.csv

@@ -9,6 +9,7 @@ gr-ignore-annotations
 gr-init-deploy
 gr-one-pod
 gr-scaling
+gr-self-healing
 gr-tls-cert-manager
 haproxy
 init-deploy

e2e-tests/tests/gr-self-healing/00-assert.yaml

@@ -0,0 +1,26 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: perconaservermysqls.ps.percona.com
+spec:
+  group: ps.percona.com
+  names:
+    kind: PerconaServerMySQL
+    listKind: PerconaServerMySQLList
+    plural: perconaservermysqls
+    shortNames:
+    - ps
+    singular: perconaservermysql
+  scope: Namespaced
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: check-operator-deploy-status
+timeout: 120
+commands:
+  - script: kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1

e2e-tests/tests/gr-self-healing/00-deploy-operator.yaml

@@ -0,0 +1,14 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      deploy_operator
+      deploy_non_tls_cluster_secrets
+      deploy_tls_cluster_secrets
+      deploy_client

e2e-tests/tests/gr-self-healing/01-assert.yaml

@@ -0,0 +1,27 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: chaos-controller-manager
+spec:
+  replicas: 3
+status:
+  availableReplicas: 3
+  readyReplicas: 3
+  replicas: 3
+  updatedReplicas: 3
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: chaos-daemon
+status:
+  currentNumberScheduled: 3
+  desiredNumberScheduled: 3
+  numberAvailable: 3
+  numberMisscheduled: 0
+  numberReady: 3
+  updatedNumberScheduled: 3

e2e-tests/tests/gr-self-healing/01-deploy-chaos-mesh.yaml

@@ -0,0 +1,11 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      deploy_chaos_mesh