K8SPS-160: SmartUpdate (#405)

inelpandzic · github-actions[bot] · web-flow · commit 6f7dd41f193f · 2023-08-10T14:29:43.000+02:00
* Add UpdateStrategy option.

* Apply appropriate updateStrategy to statefulSets.

* Implement SmartUpdate feature.

* Improve implementation and set updateStrategy to SmartUpdate by default
in CR.

* Update tests.

* Implement check if node is ready (status is active/online).

* Fix lint issue.

* Fix test.

* Update e2e tests.

* Update pkg/mysql/mysql.go

Co-authored-by: github-actions[bot] &lt;41898282+github-actions[bot]@users.noreply.github.com&gt;

* Improved smartUpdate algorithm.

* Fix haproxy e2e test.

* Add e2e test and improve smart update logic.

* Fix updating status, initializing state for the whole duration of sts
ugrade.

* Update e2e test.

* Reorder tests.

* Minor refactor.

* Delete pod a wait for sts.Status.ReadyReplicas to be updated properly.

* Refactor appStatus.

---------

Co-authored-by: github-actions[bot] &lt;41898282+github-actions[bot]@users.noreply.github.com&gt;
diff --git a/api/v1alpha1/perconaservermysql_types.go b/api/v1alpha1/perconaservermysql_types.go
@@ -29,6 +29,7 @@ import (
 	"github.com/percona/percona-server-mysql-operator/pkg/version"
 
 	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -41,23 +42,24 @@ import (
 
 // PerconaServerMySQLSpec defines the desired state of PerconaServerMySQL
 type PerconaServerMySQLSpec struct {
-	CRVersion             string           `json:"crVersion,omitempty"`
-	Pause                 bool             `json:"pause,omitempty"`
-	SecretsName           string           `json:"secretsName,omitempty"`
-	SSLSecretName         string           `json:"sslSecretName,omitempty"`
-	SSLInternalSecretName string           `json:"sslInternalSecretName,omitempty"`
-	AllowUnsafeConfig     bool             `json:"allowUnsafeConfigurations,omitempty"`
-	InitImage             string           `json:"initImage,omitempty"`
-	IgnoreAnnotations     []string         `json:"ignoreAnnotations,omitempty"`
-	IgnoreLabels          []string         `json:"ignoreLabels,omitempty"`
-	MySQL                 MySQLSpec        `json:"mysql,omitempty"`
-	Orchestrator          OrchestratorSpec `json:"orchestrator,omitempty"`
-	PMM                   *PMMSpec         `json:"pmm,omitempty"`
-	Backup                *BackupSpec      `json:"backup,omitempty"`
-	Proxy                 ProxySpec        `json:"proxy,omitempty"`
-	TLS                   *TLSSpec         `json:"tls,omitempty"`
-	Toolkit               *ToolkitSpec     `json:"toolkit,omitempty"`
-	UpgradeOptions        UpgradeOptions   `json:"upgradeOptions,omitempty"`
+	CRVersion             string                               `json:"crVersion,omitempty"`
+	Pause                 bool                                 `json:"pause,omitempty"`
+	SecretsName           string                               `json:"secretsName,omitempty"`
+	SSLSecretName         string                               `json:"sslSecretName,omitempty"`
+	SSLInternalSecretName string                               `json:"sslInternalSecretName,omitempty"`
+	AllowUnsafeConfig     bool                                 `json:"allowUnsafeConfigurations,omitempty"`
+	InitImage             string                               `json:"initImage,omitempty"`
+	IgnoreAnnotations     []string                             `json:"ignoreAnnotations,omitempty"`
+	IgnoreLabels          []string                             `json:"ignoreLabels,omitempty"`
+	MySQL                 MySQLSpec                            `json:"mysql,omitempty"`
+	Orchestrator          OrchestratorSpec                     `json:"orchestrator,omitempty"`
+	PMM                   *PMMSpec                             `json:"pmm,omitempty"`
+	Backup                *BackupSpec                          `json:"backup,omitempty"`
+	Proxy                 ProxySpec                            `json:"proxy,omitempty"`
+	TLS                   *TLSSpec                             `json:"tls,omitempty"`
+	Toolkit               *ToolkitSpec                         `json:"toolkit,omitempty"`
+	UpgradeOptions        UpgradeOptions                       `json:"upgradeOptions,omitempty"`
+	UpdateStrategy        appsv1.StatefulSetUpdateStrategyType `json:"updateStrategy,omitempty"`
 }
 
 type TLSSpec struct {
@@ -674,6 +676,12 @@ func (cr *PerconaServerMySQL) CheckNSetDefaults(ctx context.Context, serverVersi
 		cr.Spec.SSLSecretName = cr.Name + "-ssl"
 	}
 
+	if cr.Spec.UpdateStrategy == SmartUpdateStatefulSetStrategyType &&
+		!cr.HAProxyEnabled() &&
+		!cr.RouterEnabled() {
+		return errors.Errorf("MySQL Router or HAProxy should be enabled if SmartUpdate set")
+	}
+
 	return nil
 }
 
@@ -893,6 +901,9 @@ func init() {
 	SchemeBuilder.Register(&PerconaServerMySQL{}, &PerconaServerMySQLList{})
 }
 
+// SmartUpdateStatefulSetStrategyType
+const SmartUpdateStatefulSetStrategyType appsv1.StatefulSetUpdateStrategyType = "SmartUpdate"
+
 type UpgradeOptions struct {
 	VersionServiceEndpoint string `json:"versionServiceEndpoint,omitempty"`
 	Apply                  string `json:"apply,omitempty"`
diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml
@@ -7135,6 +7135,8 @@ spec:
                 required:
                 - image
                 type: object
+              updateStrategy:
+                type: string
               upgradeOptions:
                 properties:
                   apply:
diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml
@@ -8747,6 +8747,8 @@ spec:
                 required:
                 - image
                 type: object
+              updateStrategy:
+                type: string
               upgradeOptions:
                 properties:
                   apply:
diff --git a/deploy/cr.yaml b/deploy/cr.yaml
@@ -11,6 +11,7 @@ spec:
   crVersion: 0.6.0
   secretsName: cluster1-secrets
   sslSecretName: cluster1-ssl
+  updateStrategy: SmartUpdate
   upgradeOptions:
     versionServiceEndpoint: https://check.percona.com
     apply: disabled
diff --git a/deploy/crd.yaml b/deploy/crd.yaml
@@ -8747,6 +8747,8 @@ spec:
                 required:
                 - image
                 type: object
+              updateStrategy:
+                type: string
               upgradeOptions:
                 properties:
                   apply:
diff --git a/e2e-tests/run-distro.csv b/e2e-tests/run-distro.csv
@@ -16,5 +16,6 @@ one-pod
 scaling
 service-per-pod
 sidecars
+smart-update
 tls-cert-manager
 users
diff --git a/e2e-tests/run-minikube.csv b/e2e-tests/run-minikube.csv
@@ -13,6 +13,7 @@ haproxy
 init-deploy
 one-pod
 sidecars
+smart-update
 tls-cert-manager
 users
 version-service
diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv
@@ -20,6 +20,7 @@ one-pod
 scaling
 service-per-pod
 sidecars
+smart-update
 tls-cert-manager
 users
 version-service
diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv
@@ -19,6 +19,7 @@ one-pod
 scaling
 service-per-pod
 sidecars
+smart-update
 tls-cert-manager
 users
 version-service
diff --git a/e2e-tests/tests/haproxy/01-create-cluster.yaml b/e2e-tests/tests/haproxy/01-create-cluster.yaml
@@ -10,6 +10,7 @@ commands:
 
       get_cr \
         | yq eval '.spec.allowUnsafeConfigurations=false' - \
+        | yq eval '.spec.updateStrategy="RollingUpdate"' - \
         | yq eval '.spec.mysql.clusterType="async"' - \
         | yq eval '.spec.orchestrator.enabled=true' - \
         | yq eval '.spec.proxy.haproxy.enabled=true' - \
diff --git a/e2e-tests/tests/limits/01-assert.yaml b/e2e-tests/tests/limits/01-assert.yaml
@@ -214,7 +214,7 @@ spec:
       - emptyDir: {}
         name: backup-logs
   updateStrategy:
-    type: RollingUpdate
+    type: OnDelete
   volumeClaimTemplates:
   - apiVersion: v1
     kind: PersistentVolumeClaim
diff --git a/e2e-tests/tests/limits/03-assert.yaml b/e2e-tests/tests/limits/03-assert.yaml
@@ -214,7 +214,7 @@ spec:
       - emptyDir: {}
         name: backup-logs
   updateStrategy:
-    type: RollingUpdate
+    type: OnDelete
   volumeClaimTemplates:
   - apiVersion: v1
     kind: PersistentVolumeClaim
diff --git a/e2e-tests/tests/limits/05-assert.yaml b/e2e-tests/tests/limits/05-assert.yaml
@@ -212,7 +212,7 @@ spec:
       - emptyDir: {}
         name: backup-logs
   updateStrategy:
-    type: RollingUpdate
+    type: OnDelete
   volumeClaimTemplates:
   - apiVersion: v1
     kind: PersistentVolumeClaim
diff --git a/e2e-tests/tests/smart-update/00-assert.yaml b/e2e-tests/tests/smart-update/00-assert.yaml
@@ -0,0 +1,109 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: perconaservermysqls.ps.percona.com
+spec:
+  group: ps.percona.com
+  names:
+    kind: PerconaServerMySQL
+    listKind: PerconaServerMySQLList
+    plural: perconaservermysqls
+    shortNames:
+    - ps
+    singular: perconaservermysql
+  scope: Namespaced
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: percona-server-mysql-operator
+status:
+  availableReplicas: 1
+  observedGeneration: 1
+  readyReplicas: 1
+  replicas: 1
+  updatedReplicas: 1
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: mysql-client
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: percona-version-service
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      name: percona-version-service
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        name: percona-version-service
+    spec:
+      containers:
+        - env:
+            - name: SERVE_HTTP
+              value: "true"
+          imagePullPolicy: Always
+          name: percona-version-service
+          ports:
+            - containerPort: 11000
+              name: http
+              protocol: TCP
+          resources: {}
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          volumeMounts:
+            - mountPath: /sources/operator.9.9.9.ps-operator.dep.json
+              name: versions
+              subPath: operator.9.9.9.ps-operator.dep.json
+            - mountPath: /sources/operator.9.9.9.ps-operator.json
+              name: versions
+              subPath: operator.9.9.9.ps-operator.json
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30
+      volumes:
+        - configMap:
+            defaultMode: 420
+            name: versions
+          name: versions
+status:
+  readyReplicas: 1
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    name: percona-version-service
+  name: percona-version-service
+spec:
+  ipFamilies:
+    - IPv4
+  ipFamilyPolicy: SingleStack
+  ports:
+    - port: 80
+      protocol: TCP
+      targetPort: 11000
+  selector:
+    name: percona-version-service
+  sessionAffinity: None
+  type: ClusterIP
+status:
+  loadBalancer: {}
diff --git a/e2e-tests/tests/smart-update/00-deploy-operator.yaml b/e2e-tests/tests/smart-update/00-deploy-operator.yaml
@@ -0,0 +1,15 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      deploy_operator
+      deploy_non_tls_cluster_secrets
+      deploy_tls_cluster_secrets
+      deploy_client
+      deploy_version_service
diff --git a/e2e-tests/tests/smart-update/01-assert.yaml b/e2e-tests/tests/smart-update/01-assert.yaml
@@ -0,0 +1,44 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 420
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  generation: 1
+  name: smart-update-mysql
+spec:
+  updateStrategy:
+    type: OnDelete
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  generation: 1
+  name: smart-update-orc
+spec:
+  updateStrategy:
+    type: RollingUpdate
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: smart-update-haproxy
+spec:
+  updateStrategy:
+    type: RollingUpdate
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
diff --git a/e2e-tests/tests/smart-update/01-create-cluster.yaml b/e2e-tests/tests/smart-update/01-create-cluster.yaml
@@ -0,0 +1,18 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      get_cr \
+        | yq eval '.spec.mysql.clusterType="async"' - \
+        | yq eval '.spec.updateStrategy="SmartUpdate"' - \
+        | yq eval '.spec.upgradeOptions.apply="disabled"' - \
+        | yq eval '.spec.mysql.image="percona/percona-server:8.0.28-19"' - \
+        | yq eval '.spec.backup.enabled=false' - \
+        | yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://percona-version-service:80"' - \
+        | kubectl -n "${NAMESPACE}" apply -f -
diff --git a/e2e-tests/tests/smart-update/02-check-smart-update.yaml b/e2e-tests/tests/smart-update/02-check-smart-update.yaml
@@ -0,0 +1,30 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      initialPrimary=$(get_primary_from_haproxy ${test_name}-haproxy-0)
+
+      # Upgrade cluster
+      kubectl -n "${NAMESPACE}" patch ps "$(get_cluster_name)" \
+      	--type json \
+      	-p '[{"op": "replace", "path": "/spec/mysql/image", "value": "percona/percona-server:8.0.28-20"}]'
+
+      wait_cluster_consistency_async "${test_name}" "3" "3"
+
+      pods=$(kubectl get pods -n "${NAMESPACE}" --sort-by=.status.startTime --no-headers --selector=app.kubernetes.io/component=mysql -ojson | jq  --raw-output '.items[].metadata.name')
+
+      lastRestarted=$(echo $pods | awk '{print $NF}')
+
+      if [ "$lastRestarted" == "$initialPrimary" ]; then
+        exit 0
+      fi
+
+      echo "last restarted pod is not initial primary"
+      exit 1
+    timeout: 480
+
diff --git a/e2e-tests/tests/smart-update/03-drop-finalizer.yaml b/e2e-tests/tests/smart-update/03-drop-finalizer.yaml
diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go
diff --git a/pkg/controller/ps/controller_test.go b/pkg/controller/ps/controller_test.go
diff --git a/pkg/controller/ps/status.go b/pkg/controller/ps/status.go
diff --git a/pkg/controller/ps/status_test.go b/pkg/controller/ps/status_test.go
diff --git a/pkg/controller/ps/upgrade.go b/pkg/controller/ps/upgrade.go
diff --git a/pkg/haproxy/haproxy.go b/pkg/haproxy/haproxy.go
diff --git a/pkg/mysql/mysql.go b/pkg/mysql/mysql.go
diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go
