Merge pull request #1146 from oracle/endtoend

 fixed intermitent failures for delayed alert fire

Author: rjeberhard

integration-tests/src/test/java/oracle/kubernetes/operator/ItMonitoringExporter.java

@@ -852,13 +852,12 @@ private void fireAlert() throws Exception {
 
     TestUtils.checkPodReady("domain1-admin-server", "default");
     TestUtils.checkPodReady("domain1-managed-server-1", "default");
-    //sleep for 2 min to fire alert
-    Thread.sleep(120000);
+
     String webhookPod = getPodName("webhook", "webhook");
     String command = "kubectl -n webhook logs " + webhookPod;
-    ExecResult webhookResult = TestUtils.exec(command);
+
+    ExecResult webhookResult = TestUtils.checkAnyCmdInLoop(command, "Some WLS cluster has only one running server for more than 1 minutes");
     logger.info(" webhook log " + webhookResult.stdout());
-    assertTrue(webhookResult.stdout().contains("Some WLS cluster has only one running server for more than 1 minutes"));
   }
 
   private static String getPodName(String app, String namespace) throws Exception {

integration-tests/src/test/java/oracle/kubernetes/operator/utils/TestUtils.java

@@ -1224,6 +1224,49 @@ private static KeyStore createKeyStore(Operator operator) throws Exception {
     return myKeyStore;
   }
 
+  /**
+   *
+   * @param cmd command to run in the loop
+   * @param matchStr expected string to match in the output
+   * @return ExecResult object containing command output info
+   * @throws Exception exception if fails to execute
+   */
+  public static ExecResult checkAnyCmdInLoop(String cmd, String matchStr)
+          throws Exception {
+    int i = 0;
+    ExecResult result = null;
+    while (i < BaseTest.getMaxIterationsPod()) {
+      result = ExecCommand.exec(cmd);
+
+      if (result.exitValue() != 0
+              || (result.exitValue() == 0 && !result.stdout().contains(matchStr))) {
+        logger.info("Output for " + cmd + "\n" + result.stdout() + "\n " + result.stderr());
+        // check for last iteration
+        if (i == (BaseTest.getMaxIterationsPod() - 1)) {
+          throw new RuntimeException(
+                  "FAILURE: expected output " + matchStr + " from command " + cmd + " is not receieved, exiting!");
+        }
+        logger.info(
+                "did not receive the expected output "
+                        + matchStr
+                        + "from command " + cmd + " Ite ["
+                        + i
+                        + "/"
+                        + BaseTest.getMaxIterationsPod()
+                        + "], sleeping "
+                        + BaseTest.getWaitTimePod()
+                        + " seconds more");
+
+        Thread.sleep(BaseTest.getWaitTimePod() * 1000);
+        i++;
+      } else {
+        logger.info("Command " + cmd + " is successful");
+        break;
+      }
+    }
+    return result;
+  }
+
   public static void checkCmdInLoop(String cmd, String matchStr, String k8sObjName)
       throws Exception {
     int i = 0;