K8SPS-360: Orc downscale outdated services (#714)

* Remove outdated orc services on downscale.

* Refactor and log.

* Check if cluster is paused.

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

Author: inelpandzic

pkg/controller/ps/controller.go

@@ -959,16 +959,51 @@ func (r *PerconaServerMySQLReconciler) cleanupMysql(ctx context.Context, cr *api
 }
 
 func (r *PerconaServerMySQLReconciler) cleanupOrchestrator(ctx context.Context, cr *apiv1alpha1.PerconaServerMySQL) error {
+	orcExposer := orchestrator.Exposer(*cr)
+
 	if !cr.OrchestratorEnabled() {
 		if err := r.Delete(ctx, orchestrator.StatefulSet(cr, "", "")); err != nil && !k8serrors.IsNotFound(err) {
 			return errors.Wrap(err, "failed to delete orchestrator statefulset")
 		}
 
-		orcExposer := orchestrator.Exposer(*cr)
 		if err := r.cleanupOutdatedServices(ctx, &orcExposer, cr.Namespace); err != nil {
 			return errors.Wrap(err, "cleanup Orchestrator services")
 		}
+
+		return nil
+	}
+
+	if cr.Spec.Pause {
+		return nil
+	}
+
+	svcLabels := orcExposer.Labels()
+	svcLabels[naming.LabelExposed] = "true"
+	services, err := k8s.ServicesByLabels(ctx, r.Client, svcLabels, cr.Namespace)
+	if err != nil {
+		return errors.Wrap(err, "get exposed services")
+	}
+
+	if len(services) == int(orcExposer.Size()) {
+		return nil
+	}
+
+	outdatedSvcs := make(map[string]struct{}, len(services))
+	for i := len(services) - 1; i >= int(orcExposer.Size()); i-- {
+		outdatedSvcs[orchestrator.PodName(cr, i)] = struct{}{}
+	}
+
+	for _, svc := range services {
+		if _, ok := outdatedSvcs[svc.Name]; !ok {
+			continue
+		}
+
+		logf.FromContext(ctx).Info("Deleting outdated orchestrator service", "service", svc.Name)
+		if err := r.Client.Delete(ctx, &svc); err != nil && !k8serrors.IsNotFound(err) {
+			return errors.Wrapf(err, "delete Service/%s", svc.Name)
+		}
 	}
+
 	return nil
 }