Fix intermittent ItMultiDomainModelsWithLoadBalancer.testLivenessProbe failure in release/3.4 nightly (#3509)

* Fix intermittent ItMultiDomainModelsWithLoadBalancer.testLivenessProbe failure in release/3.4 nightly

Author: xiancao

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMultiDomainModelsWithLoadBalancer.java

@@ -71,9 +71,11 @@
 import static oracle.weblogic.kubernetes.actions.TestActions.startDomain;
 import static oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes.copyFileToPod;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.adminNodePortAccessible;
+import static oracle.weblogic.kubernetes.assertions.TestAssertions.isNginxReady;
 import static oracle.weblogic.kubernetes.assertions.impl.Domain.doesDomainExist;
 import static oracle.weblogic.kubernetes.utils.ApplicationUtils.callWebAppAndCheckForServerNameInResponse;
 import static oracle.weblogic.kubernetes.utils.ApplicationUtils.callWebAppAndWaitTillReady;
+import static oracle.weblogic.kubernetes.utils.CommonLBTestUtils.checkIngressReady;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getNextFreePort;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.scaleAndVerifyCluster;
@@ -151,6 +153,7 @@ class ItMultiDomainModelsWithLoadBalancer {
   private static String opServiceAccount = null;
   private static NginxParams nginxHelmParams = null;
   private static int nodeportshttp = 0;
+  private static int nodeportshttps = 0;
   private static int externalRestHttpsPort = 0;
   private static List<Domain> domains = new ArrayList<>();
   private static LoggingFacade logger = null;
@@ -160,6 +163,7 @@ class ItMultiDomainModelsWithLoadBalancer {
   private static String miiDomainNegativeNamespace = null;
   private static String miiImage = null;
   private static String encryptionSecretName = "encryptionsecret";
+  private static String nginxNamespace = null;
   private String curlCmd = null;
 
   /**
@@ -182,7 +186,7 @@ public static void initAll(@Namespaces(6) List<String> namespaces) {
     // get a unique NGINX namespace
     logger.info("Get a unique namespace for NGINX");
     assertNotNull(namespaces.get(1), "Namespace list is null");
-    String nginxNamespace = namespaces.get(1);
+    nginxNamespace = namespaces.get(1);
 
     // get unique namespaces for three different type of domains
     logger.info("Getting unique namespaces for three different type of domains");
@@ -222,6 +226,8 @@ public static void initAll(@Namespaces(6) List<String> namespaces) {
       logger.info("NGINX service name: {0}", nginxServiceName);
       nodeportshttp = getServiceNodePort(nginxNamespace, nginxServiceName, "http");
       logger.info("NGINX http node port: {0}", nodeportshttp);
+      nodeportshttps = getServiceNodePort(nginxNamespace, nginxServiceName, "https");
+      logger.info("NGINX https node port: {0}", nodeportshttps);
     }
   }
 
@@ -488,11 +494,22 @@ void testLivenessProbe() {
           domainNamespace);
     }
 
+    // check the NGINX pod is ready.
+    testUntil(isNginxReady(nginxNamespace),
+        logger,
+        "Nginx is ready in namespace {0}",
+        nginxNamespace);
+
+    // check the ingress is ready
+    String ingressHost = domainUid + "." + domainNamespace + "." + clusterName + ".test";
+    logger.info("Checking for the ingress is ready");
+    checkIngressReady(true, ingressHost, false, nodeportshttp, nodeportshttps, "");
+
     //access application in managed servers through NGINX load balancer
     logger.info("Accessing the sample app through NGINX load balancer");
     String curlCmd = generateCurlCmd(domainUid, domainNamespace, clusterName, SAMPLE_APP_CONTEXT_ROOT);
     List<String> managedServers = listManagedServersBeforeScale(numClusters, clusterName, replicaCount);
-    testUntil(() -> callWebAppAndCheckForServerNameInResponse(curlCmd, managedServers, 20),
+    testUntil(withLongRetryPolicy, () -> callWebAppAndCheckForServerNameInResponse(curlCmd, managedServers, 20),
         logger,
         "NGINX can access the test web app from all managed servers {0} in the domain",
         managedServers);
@@ -684,16 +701,19 @@ void testMiiMultiClustersRollingRestart() {
     testUntil(
         checkDomainEvent(opNamespace, miiDomainNamespace, miiDomainUid, DOMAIN_CHANGED, "Normal", timestamp),
         logger,
-        "domain event {0} to be logged",
-        DOMAIN_CHANGED);
+        "domain event {0} to be logged in namespace {1}",
+        DOMAIN_CHANGED,
+        miiDomainNamespace);
 
     // wait for longer time for DomainCompleted event
     testUntil(
         withLongRetryPolicy,
         checkDomainEvent(opNamespace, miiDomainNamespace, miiDomainUid, DOMAIN_PROCESSING_COMPLETED,
             "Normal", timestamp),
         logger,
-        DOMAIN_PROCESSING_COMPLETED);
+        "domain event {0} to be logged in namespace {1}",
+        DOMAIN_PROCESSING_COMPLETED,
+        miiDomainNamespace);
 
     // Verify that pod termination and started events are logged only once for each managed server in each cluster
     for (int i = 1; i <= NUMBER_OF_CLUSTERS_MIIDOMAIN; i++) {
@@ -703,20 +723,25 @@ void testMiiMultiClustersRollingRestart() {
 
         logger.info("Checking that managed server pod {0} is terminated and restarted once in namespace {1}",
             managedServerPodName, miiDomainNamespace);
+        String stopEventMsg = "Stopping container weblogic-server";
         testUntil(
             withLongRetryPolicy,
-            checkPodEventLoggedOnce(miiDomainNamespace, managedServerPodName, POD_TERMINATED, timestamp),
+            checkPodEventLoggedOnce(miiDomainNamespace, managedServerPodName, POD_TERMINATED, stopEventMsg, timestamp),
             logger,
-            "event {0} to be logged for pod {1}",
+            "event {0} to be logged for pod {1} in namespace {2}",
             POD_TERMINATED,
-            managedServerPodName);
+            managedServerPodName,
+            miiDomainNamespace);
+
+        String startedEventMsg = "Started container weblogic-server";
         testUntil(
             withLongRetryPolicy,
-            checkPodEventLoggedOnce(miiDomainNamespace, managedServerPodName, POD_STARTED, timestamp),
+            checkPodEventLoggedOnce(miiDomainNamespace, managedServerPodName, POD_STARTED, startedEventMsg, timestamp),
             logger,
-            "event {0} to be logged for pod {1}",
+            "event {0} to be logged for pod {1} in namespace {2}",
             POD_STARTED,
-            managedServerPodName);
+            managedServerPodName,
+            miiDomainNamespace);
       }
     }
 

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonLBTestUtils.java

@@ -608,7 +608,10 @@ public static void checkIngressReady(boolean isHostRouting, String ingressHost,
         }
       }
       getLogger().info("Executing curl command {0}", curlCmd);
-      assertTrue(callWebAppAndWaitTillReady(curlCmd, 60));
+      testUntil(() -> callWebAppAndWaitTillReady(curlCmd, 5),
+          getLogger(),
+          "ingress can access ready app using {0}",
+          curlCmd);
     }
   }
 

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java

@@ -406,17 +406,19 @@ public static int getEventCount(
    * @param domainNamespace namespace in which the domain exists
    * @param serverName server pod name for which event is checked
    * @param reason event to check for Started, Killing etc
+   * @param eventMsg event msg for the event
    * @param timestamp the timestamp after which to see events
    */
   public static Callable<Boolean> checkPodEventLoggedOnce(
-          String domainNamespace, String serverName, String reason, OffsetDateTime timestamp) {
+          String domainNamespace, String serverName, String reason, String eventMsg, OffsetDateTime timestamp) {
     return () -> {
       logger.info("Verifying {0} event is logged for {1} pod in the domain namespace {2}",
               reason, serverName, domainNamespace);
       try {
         return isEventLoggedOnce(serverName, Kubernetes.listNamespacedEvents(domainNamespace).stream()
                 .filter(e -> e.getInvolvedObject().getName().equals(serverName))
                 .filter(e -> e.getReason().contains(reason))
+                .filter(e -> e.getMessage().contains(eventMsg))
                 .filter(e -> isEqualOrAfter(timestamp, e)).collect(Collectors.toList()).size());
       } catch (ApiException ex) {
         Logger.getLogger(K8sEvents.class.getName()).log(Level.SEVERE, null, ex);