Merge remote-tracking branch 'origin/develop' into externalJenkins-integ-test

Author: maggiehe00

integration-tests/src/test/java/oracle/kubernetes/operator/ItElasticLogging.java

@@ -124,37 +124,60 @@ public static void staticUnPrepare() throws Exception {
       logger.info("SUCCESS");
     }
   }
-
+  
   private static void verifyElasticStackReady() throws Exception {
     // Get Logstash info
     String healthStatus = execElasticStackStatusCheck("*logstash*", "$1");
     String indexStatus = execElasticStackStatusCheck("*logstash*", "$2");
     String indexName = execElasticStackStatusCheck("*logstash*", "$3");
 
-    // Verify that the health status of Logstash
     Assume.assumeNotNull(healthStatus);
+    Assume.assumeNotNull(indexStatus);
+    Assume.assumeNotNull(indexName);
+    // Verify that the health status of Logstash
     Assume.assumeTrue(
         "Logstash is not ready!",
-        healthStatus.equalsIgnoreCase("yellow") || healthStatus.equalsIgnoreCase("green"));
+        healthStatus.equalsIgnoreCase("yellow") || 
+        healthStatus.equalsIgnoreCase("green"));
     // Verify that the index is open for use
-    Assume.assumeNotNull(indexStatus);
-    Assume.assumeTrue("Logstash index is not open!", indexStatus.equalsIgnoreCase("open"));
+    Assume.assumeTrue("Logstash index is not open!", 
+                      indexStatus.equalsIgnoreCase("open"));
     // Add the index name to a Map
-    Assume.assumeNotNull(indexName);
     testVarMap.put("indexName", indexName);
 
     // Get Kibana info
     healthStatus = execElasticStackStatusCheck("*kibana*", "$1");
     indexStatus = execElasticStackStatusCheck("*kibana*", "$2");
-
-    // Verify that the health status of Kibana
+    indexName = execElasticStackStatusCheck("*kibana*", "$3");
+    
     Assume.assumeNotNull(healthStatus);
-    Assume.assumeTrue(
-        "Kibana is not ready!",
-        healthStatus.equalsIgnoreCase("yellow") || healthStatus.equalsIgnoreCase("green"));
-    // Verify that the index is open for use
     Assume.assumeNotNull(indexStatus);
-    Assume.assumeTrue("Kibana index is not open!", indexStatus.equalsIgnoreCase("open"));
+    Assume.assumeNotNull(indexName);
+
+    //There are multiple indexes from Kibana 6.8.0
+    String[] kibanahealthStatusArr = 
+      healthStatus.split(System.getProperty("line.separator"));
+    String[] kibanaindexStatusArr = 
+      indexStatus.split(System.getProperty("line.separator"));
+    String[] kibanaindexNameArr = 
+      indexName.split(System.getProperty("line.separator"));
+    
+    for(int i = 0; i < kibanaindexStatusArr.length; i++) {
+      logger.info("Health status of " + kibanaindexNameArr[i] + 
+                  "is:" + kibanahealthStatusArr[i]);
+      logger.info("Index status of " + kibanaindexNameArr[i] + 
+                  "is:" + kibanaindexStatusArr[i]);
+      // Verify that the health status of Kibana
+      Assume.assumeTrue(
+          "Kibana is not ready!",
+          kibanahealthStatusArr[i].trim().equalsIgnoreCase("yellow") || 
+          kibanahealthStatusArr[i].trim().equalsIgnoreCase("green"));
+      // Verify that the index is open for use
+      Assume.assumeTrue("Kibana index is not open!", 
+                        kibanaindexStatusArr[i].trim().equalsIgnoreCase("open"));
+    }
+    
+    logger.info("ELK Stack is up and running and ready to use!");
   }
 
   private static String execElasticStackStatusCheck(String indexName, String varLoc)

integration-tests/src/test/java/oracle/kubernetes/operator/ItMonitoringExporter.java

@@ -852,13 +852,12 @@ private void fireAlert() throws Exception {
 
     TestUtils.checkPodReady("domain1-admin-server", "default");
     TestUtils.checkPodReady("domain1-managed-server-1", "default");
-    //sleep for 2 min to fire alert
-    Thread.sleep(120000);
+
     String webhookPod = getPodName("webhook", "webhook");
     String command = "kubectl -n webhook logs " + webhookPod;
-    ExecResult webhookResult = TestUtils.exec(command);
+
+    ExecResult webhookResult = TestUtils.checkAnyCmdInLoop(command, "Some WLS cluster has only one running server for more than 1 minutes");
     logger.info(" webhook log " + webhookResult.stdout());
-    assertTrue(webhookResult.stdout().contains("Some WLS cluster has only one running server for more than 1 minutes"));
   }
 
   private static String getPodName(String app, String namespace) throws Exception {

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Domain.java

@@ -1682,7 +1682,7 @@ private void copyCreateDomainPy() throws IOException {
                 .toPath(),
             new File(
                     BaseTest.getResultDir()
-                        + "/samples/scripts/create-fmw-infrastructure-domain/common/createFMWDomain.py")
+                        + "/samples/scripts/create-fmw-infrastructure-domain/domain-home-on-pv/common/createFMWDomain.py")
                 .toPath(),
             StandardCopyOption.REPLACE_EXISTING);
       } else {
@@ -1721,7 +1721,7 @@ private String prepareCmdToCallCreateDomainScript(String outputDir) {
           .append(" -k -i ");
     } else if (domainMap.containsKey("rcuDatabaseURL")) {
       createDomainScriptCmd.append(
-          "/samples/scripts/create-fmw-infrastructure-domain/create-domain.sh -v -i ");
+          "/samples/scripts/create-fmw-infrastructure-domain/domain-home-on-pv/create-domain.sh -v -i ");
     } else {
       createDomainScriptCmd.append(
           "/samples/scripts/create-weblogic-domain/domain-home-on-pv/create-domain.sh -v -i ");

integration-tests/src/test/java/oracle/kubernetes/operator/utils/JrfDomain.java

@@ -69,7 +69,7 @@ private void updateDomainMapForJrf(boolean adminPortEnabled) throws Exception {
     if (adminPortEnabled) {
       String createDomainScript =
           BaseTest.getResultDir()
-              + "/samples/scripts/create-fmw-infrastructure-domain/wlst/create-domain-script.sh";
+              + "/samples/scripts/create-fmw-infrastructure-domain/domain-home-on-pv/wlst/create-domain-script.sh";
       TestUtils.replaceStringInFile(
           createDomainScript,
           "-managedNameBase ",

integration-tests/src/test/java/oracle/kubernetes/operator/utils/TestUtils.java

@@ -1224,6 +1224,49 @@ private static KeyStore createKeyStore(Operator operator) throws Exception {
     return myKeyStore;
   }
 
+  /**
+   *
+   * @param cmd command to run in the loop
+   * @param matchStr expected string to match in the output
+   * @return ExecResult object containing command output info
+   * @throws Exception exception if fails to execute
+   */
+  public static ExecResult checkAnyCmdInLoop(String cmd, String matchStr)
+          throws Exception {
+    int i = 0;
+    ExecResult result = null;
+    while (i < BaseTest.getMaxIterationsPod()) {
+      result = ExecCommand.exec(cmd);
+
+      if (result.exitValue() != 0
+              || (result.exitValue() == 0 && !result.stdout().contains(matchStr))) {
+        logger.info("Output for " + cmd + "\n" + result.stdout() + "\n " + result.stderr());
+        // check for last iteration
+        if (i == (BaseTest.getMaxIterationsPod() - 1)) {
+          throw new RuntimeException(
+                  "FAILURE: expected output " + matchStr + " from command " + cmd + " is not receieved, exiting!");
+        }
+        logger.info(
+                "did not receive the expected output "
+                        + matchStr
+                        + "from command " + cmd + " Ite ["
+                        + i
+                        + "/"
+                        + BaseTest.getMaxIterationsPod()
+                        + "], sleeping "
+                        + BaseTest.getWaitTimePod()
+                        + " seconds more");
+
+        Thread.sleep(BaseTest.getWaitTimePod() * 1000);
+        i++;
+      } else {
+        logger.info("Command " + cmd + " is successful");
+        break;
+      }
+    }
+    return result;
+  }
+
   public static void checkCmdInLoop(String cmd, String matchStr, String k8sObjName)
       throws Exception {
     int i = 0;

integration-tests/src/test/resources/operator_elk.yaml

@@ -9,6 +9,6 @@ domainNamespaces: [ "default", "test1" ]
 externalRestEnabled: true
 javaLoggingLevel: INFO
 elkIntegrationEnabled: true
-logStashImage: "logstash:6.6.0"
+logStashImage: "logstash:6.8.0"
 elasticSearchHost: "elasticsearch.default.svc.cluster.local"
 elasticSearchPort: 9200