K8SPS-548 resolve gr-self-healing test failures (#1142)

* K8SPS-548 resolve gr-self-healing test failures

Fix two critical issues causing gr-self-healing test failures:

1. False split-brain detection due to incorrect quorum calculation
   - CheckIfInPrimaryPartition now only counts ONLINE members for quorum
   - Prevents false "possible split brain!" errors from stale UNREACHABLE entries

2. Cluster creation failures from stale group_replication_group_seeds
   - Clear persisted seeds early in bootstrap process
   - Prevents "Peer address is not valid" errors after pod deletions

* fix split-brain detecion in liveness probe

* improve test

* Update e2e-tests/functions

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Update e2e-tests/functions

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Update e2e-tests/functions

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Update e2e-tests/functions

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

---------

Co-authored-by: George Kechagias <george.kechagias@percona.com>
Co-authored-by: Ege Güneş <ege.gunes@percona.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: 4 people

cmd/bootstrap/gr/group_replication.go

@@ -408,6 +408,13 @@ func Bootstrap(ctx context.Context) error {
 		return errors.Wrap(err, "connect to local")
 	}
 
+	// Clear group_replication_group_seeds early to prevent any stale entries
+	// from causing "Peer address is not valid" errors during cluster operations
+	log.Printf("Clearing any stale group_replication_group_seeds")
+	if err := localShell.setGroupSeeds(ctx, ""); err != nil {
+		log.Printf("WARNING: failed to clear group_replication_group_seeds: %v", err)
+	}
+
 	err = localShell.configureInstance(ctx)
 	if err != nil {
 		return err
@@ -463,6 +470,7 @@ func Bootstrap(ctx context.Context) error {
 			}
 		} else {
 			log.Printf("Can't connect to any of the peers, we need to reboot")
+
 			if err := handleFullClusterCrash(ctx, mysqlshVer); err != nil {
 				return errors.Wrap(err, "handle full cluster crash")
 			}

e2e-tests/functions

@@ -1302,6 +1302,8 @@ check_primary_chaos() {
 	local chaos_type=$1
 	local ns=$2
 	local primary_before_failure=$3
+	local primary_restart_count_before=$(kubectl get pod -n ${NAMESPACE} ${primary_before_failure} -o yaml \
+		| yq '.status.containerStatuses[] | select(.name=="mysql") | .restartCount')
 
 	local chaos_name
 	case ${chaos_type} in
@@ -1330,12 +1332,19 @@ check_primary_chaos() {
 
 	wait_cluster_consistency_gr "$(get_cluster_name)" 3 3
 
-	primary_after_failure=$(get_primary_from_group_replication)
-	uri=$(get_mysqlsh_uri_for_pod ${primary_after_failure})
+	local primary_restart_count_after=$(kubectl get pod -n ${NAMESPACE} ${primary_before_failure} -o yaml \
+		| yq '.status.containerStatuses[] | select(.name=="mysql") | .restartCount')
+	local primary_after_failure=$(get_primary_from_group_replication)
+	local uri=$(get_mysqlsh_uri_for_pod ${primary_after_failure})
 	wait_until_innodb_ok ${uri}
 
-	if [[ ${primary_before_failure} == "${primary_after_failure}" ]]; then
-		echo "primary pod was not killed! something went wrong."
+	if [[ ${chaos_type} != "full-cluster-crash" && ${primary_before_failure} == "${primary_after_failure}" ]]; then
+		echo "primary pod was not changed! something went wrong re-election."
+		exit 1
+	fi
+
+	if [[ ${chaos_type} == "network" && ${primary_restart_count_before} == ${primary_restart_count_after} ]]; then
+		echo "old primary pod was not restarted! something went wrong with liveness probe."
 		exit 1
 	fi