Merge pull request #1079 from oracle/multiclusterwdt

Adding multiple dynamic clusters in a domain

Author: rjeberhard

integration-tests/src/test/java/oracle/kubernetes/operator/ITMultipleClusters.java

@@ -5,13 +5,12 @@
 
 import static org.junit.Assert.assertTrue;
 
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
 import java.util.Map;
+import java.util.logging.Level;
 import oracle.kubernetes.operator.utils.Domain;
 import oracle.kubernetes.operator.utils.K8sTestUtils;
 import oracle.kubernetes.operator.utils.Operator;
@@ -34,7 +33,7 @@ public class ITMultipleClusters extends BaseTest {
   private static final String TWO_CONFIGURED_CLUSTER_SCRIPT =
       "create-domain-two-configured-cluster.py";
   private static final String TWO_MIXED_CLUSTER_SCRIPT = "create-domain-two-mixed-cluster.py";
-  private static String template;
+  private static String customDomainTemplate;
   private static final String DOMAINUID = "twoconfigclustdomain";
 
   /**
@@ -48,8 +47,16 @@ public class ITMultipleClusters extends BaseTest {
   public static void staticPrepare() throws Exception {
     // initialize test properties and create the directories
     initialize(APP_PROPS_FILE);
-    template =
+    String template =
         BaseTest.getProjectRoot() + "/kubernetes/samples/scripts/common/domain-template.yaml";
+    String add =
+        "  - clusterName: %CLUSTER_NAME%-2\n"
+            + "    serverStartState: \"RUNNING\"\n"
+            + "    replicas: %INITIAL_MANAGED_SERVER_REPLICAS%\n";
+    customDomainTemplate = BaseTest.getResultDir() + "/customDomainTemplate.yaml";
+    Files.copy(
+        Paths.get(template), Paths.get(customDomainTemplate), StandardCopyOption.REPLACE_EXISTING);
+    Files.write(Paths.get(customDomainTemplate), add.getBytes(), StandardOpenOption.APPEND);
   }
 
   /**
@@ -89,6 +96,7 @@ public void testCreateDomainTwoConfiguredCluster() throws Exception {
       Map<String, Object> domainMap = TestUtils.loadYaml(DOMAINONPV_WLST_YAML);
       domainMap.put("domainUID", DOMAINUID);
       domainMap.put("clusterType", "CONFIGURED");
+      domainMap.put("customDomainTemplate", customDomainTemplate);
       domainMap.put(
           "createDomainPyScript",
           "integration-tests/src/test/resources/domain-home-on-pv/"
@@ -98,10 +106,17 @@ public void testCreateDomainTwoConfiguredCluster() throws Exception {
               && ((String) domainMap.get("loadBalancer")).equalsIgnoreCase("VOYAGER"))) {
         domainMap.put("voyagerWebPort", new Integer("30366"));
       }
-      addCluster2ToDomainTemplate();
       domain = TestUtils.createDomain(domainMap);
       domain.verifyDomainCreated();
-      verifyServersStatus(domain);
+      String pods[] = {
+        DOMAINUID + "-" + domain.getAdminServerName(),
+        DOMAINUID + "-managed-server",
+        DOMAINUID + "-managed-server1",
+        DOMAINUID + "-managed-server2",
+        DOMAINUID + "-new-managed-server1",
+        DOMAINUID + "-new-managed-server2",
+      };
+      verifyServersStatus(domain, pods);
       testBasicUseCases(domain);
       if (!SMOKETEST) {
         domain.testWlsLivenessProbe();
@@ -111,7 +126,6 @@ public void testCreateDomainTwoConfiguredCluster() throws Exception {
       if (domain != null && !SMOKETEST && (JENKINS || testCompletedSuccessfully)) {
         domain.destroy();
       }
-      restoreDomainTemplate();
     }
     logger.info("SUCCESS - " + testMethodName);
   }
@@ -138,6 +152,7 @@ public void testCreateDomainTwoMixedCluster() throws Exception {
     try {
       Map<String, Object> domainMap = TestUtils.loadYaml(DOMAINONPV_WLST_YAML);
       domainMap.put("domainUID", DOMAINUID);
+      domainMap.put("customDomainTemplate", customDomainTemplate);
       domainMap.put(
           "createDomainPyScript",
           "integration-tests/src/test/resources/domain-home-on-pv/" + TWO_MIXED_CLUSTER_SCRIPT);
@@ -146,10 +161,18 @@ public void testCreateDomainTwoMixedCluster() throws Exception {
               && ((String) domainMap.get("loadBalancer")).equalsIgnoreCase("VOYAGER"))) {
         domainMap.put("voyagerWebPort", new Integer("30377"));
       }
-      addCluster2ToDomainTemplate();
       domain = TestUtils.createDomain(domainMap);
       domain.verifyDomainCreated();
-      verifyServersStatus(domain);
+      String pods[] = {
+        DOMAINUID + "-" + domain.getAdminServerName(),
+        DOMAINUID + "-managed-server",
+        DOMAINUID + "-managed-server1",
+        DOMAINUID + "-managed-server2",
+        DOMAINUID + "-new-managed-server1",
+        DOMAINUID + "-new-managed-server2",
+      };
+      verifyServersStatus(domain, pods);
+
       testBasicUseCases(domain);
       if (!SMOKETEST) {
         domain.testWlsLivenessProbe();
@@ -159,58 +182,73 @@ public void testCreateDomainTwoMixedCluster() throws Exception {
       if (domain != null && !SMOKETEST && (JENKINS || testCompletedSuccessfully)) {
         domain.destroy();
       }
-      restoreDomainTemplate();
     }
     logger.info("SUCCESS - " + testMethodName);
   }
 
   /**
-   * Append a second cluster to the domain template
+   * Create 2 dynamic clusters in a domain using WDT in inimage each having 2 managed servers.
+   * Verify the managed servers are running and verify the basic use cases.
    *
-   * @throws IOException when append fails
+   * @throws Exception
    */
-  private void addCluster2ToDomainTemplate() throws IOException {
-    String add =
-        "  - clusterName: %CLUSTER_NAME%-2\n"
-            + "    serverStartState: \"RUNNING\"\n"
-            + "    replicas: %INITIAL_MANAGED_SERVER_REPLICAS%\n";
-    logger.info("Making a backup of the domain template file:" + template);
-    if (!Files.exists(Paths.get(template + ".org"))) {
-      Files.copy(Paths.get(template), Paths.get(template + ".org"));
+  @Test
+  public void testCreateDomainTwoClusterWDTInImage() throws Exception {
+    String DOMAINUID = "twoclusterdomainwdt";
+    String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
+    logTestBegin(testMethodName);
+    logger.info("Creating Operator & waiting for the script to complete execution");
+    if (operator1 == null) {
+      operator1 = TestUtils.createOperator(OPERATOR1_YAML);
     }
-    Files.write(Paths.get(template), add.getBytes(), StandardOpenOption.APPEND);
-    byte[] readAllBytes = Files.readAllBytes(Paths.get(template));
-    logger.info(new String(readAllBytes, StandardCharsets.UTF_8));
-  }
-
-  /**
-   * Restore the domain template to original state when test is finished
-   *
-   * @throws IOException
-   */
-  private void restoreDomainTemplate() throws IOException {
-    Files.copy(
-        Paths.get(template + ".org"), Paths.get(template), StandardCopyOption.REPLACE_EXISTING);
-    Files.delete(Paths.get(template + ".org"));
+    Domain domain = null;
+    boolean testCompletedSuccessfully = false;
+    try {
+      Map<String, Object> domainMap = TestUtils.loadYaml(DOMAININIMAGE_WDT_YAML);
+      domainMap.put("domainUID", DOMAINUID);
+      domainMap.put("customDomainTemplate", customDomainTemplate);
+      if ((System.getenv("LB_TYPE") != null && System.getenv("LB_TYPE").equalsIgnoreCase("VOYAGER"))
+          || (domainMap.containsKey("loadBalancer")
+              && ((String) domainMap.get("loadBalancer")).equalsIgnoreCase("VOYAGER"))) {
+        domainMap.put("voyagerWebPort", new Integer("30377"));
+      }
+      domainMap.put(
+          "customWdtTemplate",
+          BaseTest.getProjectRoot()
+              + "/integration-tests/src/test/resources/multipleclusters/wdtmultipledynclusters.yml");
+      domain = TestUtils.createDomain(domainMap);
+      domain.verifyDomainCreated();
+      String pods[] = {
+        DOMAINUID + "-" + domain.getAdminServerName(),
+        DOMAINUID + "-managed-server1",
+        DOMAINUID + "-managed-server2",
+        DOMAINUID + "-managed-server-21",
+        DOMAINUID + "-managed-server-22",
+      };
+      verifyServersStatus(domain, pods);
+      testBasicUseCases(domain);
+      if (!SMOKETEST) {
+        domain.testWlsLivenessProbe();
+      }
+      testCompletedSuccessfully = true;
+    } finally {
+      if (domain != null && !SMOKETEST && (JENKINS || testCompletedSuccessfully)) {
+        domain.destroy();
+      }
+    }
+    logger.log(Level.INFO, "SUCCESS - {0}", testMethodName);
   }
 
   /**
    * Verifies all of the servers in the cluster are in Running status
    *
    * @param Domain
+   * @param String array pod names to check the status for
    */
-  private void verifyServersStatus(Domain domain) {
+  private void verifyServersStatus(Domain domain, String[] pods) {
     K8sTestUtils testUtil = new K8sTestUtils();
     String domain1LabelSelector = String.format("weblogic.domainUID in (%s)", DOMAINUID);
     String namespace = domain.getDomainNS();
-    String pods[] = {
-      DOMAINUID + "-" + domain.getAdminServerName(),
-      DOMAINUID + "-managed-server",
-      DOMAINUID + "-managed-server1",
-      DOMAINUID + "-managed-server2",
-      DOMAINUID + "-new-managed-server1",
-      DOMAINUID + "-new-managed-server2",
-    };
     for (String pod : pods) {
       assertTrue(
           pod + " Pod not running", testUtil.isPodRunning(namespace, domain1LabelSelector, pod));

integration-tests/src/test/java/oracle/kubernetes/operator/ITOperator.java

@@ -482,7 +482,12 @@ public void testDomainInImageUsingWDT() throws Exception {
     Domain domain = null;
     boolean testCompletedSuccessfully = false;
     try {
-      domain = TestUtils.createDomain(DOMAININIMAGE_WDT_YAML);
+      Map<String, Object> domainMap = TestUtils.loadYaml(DOMAININIMAGE_WDT_YAML);
+      domainMap.put(
+          "customWdtTemplate",
+          BaseTest.getProjectRoot()
+              + "/integration-tests/src/test/resources/wdt/config.cluster.topology.yaml");
+      domain = TestUtils.createDomain(domainMap);
       domain.verifyDomainCreated();
 
       testBasicUseCases(domain);

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Domain.java

@@ -9,6 +9,7 @@
 import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -21,6 +22,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.StringTokenizer;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import javax.jms.ConnectionFactory;
 import javax.jms.QueueConnection;
@@ -1156,6 +1158,8 @@ protected void callCreateDomainScript(String outputDir) throws Exception {
       gitCloneDockerImagesSample();
     }
 
+    copyDomainTemplate(domainMap);
+
     // copy create domain py script if domain map contains createDomainPyScript
     copyCreateDomainPy();
 
@@ -1522,6 +1526,20 @@ protected void initialize(Map<String, Object> inputDomainMap) throws Exception {
     createConfigMapAndSecretForSitConfig();
   }
 
+  private void copyDomainTemplate(Map<String, Object> inputDomainMap) throws IOException {
+    if (inputDomainMap.containsKey("customDomainTemplate")) {
+      Files.copy(
+          Paths.get((String) inputDomainMap.get("customDomainTemplate")),
+          Paths.get(BaseTest.getResultDir() + "/samples/scripts/common/domain-template.yaml"),
+          StandardCopyOption.REPLACE_EXISTING);
+    }
+    logger.log(Level.FINEST, "Domain Template");
+    byte[] readAllBytes =
+        Files.readAllBytes(
+            Paths.get(BaseTest.getResultDir() + "/samples/scripts/common/domain-template.yaml"));
+    logger.log(Level.FINEST, new String(readAllBytes, StandardCharsets.UTF_8));
+  }
+
   private String getNodeHost() throws Exception {
     String cmd =
         "kubectl describe pod "
@@ -1740,23 +1758,25 @@ private void changeClusterTypeInCreateDomainJobTemplate() throws Exception {
 
     // change CLUSTER_TYPE to CONFIGURED in create-domain-job-template.yaml for configured cluster
     // as samples only support DYNAMIC cluster
-    if (clusterType.equalsIgnoreCase("CONFIGURED")) {
-
-      // domain in image
-      if (domainMap.containsKey("domainHomeImageBase")
-          && domainHomeImageBuildPath.contains("wdt")) {
-        TestUtils.copyFile(
-            BaseTest.getProjectRoot()
-                + "/integration-tests/src/test/resources/wdt/config.cluster.topology.yaml",
-            BaseTest.getResultDir()
-                + "/docker-images/OracleWebLogic/samples/12213-domain-home-in-image-wdt/simple-topology.yaml");
-      } else {
-        // domain on pv
-        StringBuffer createDomainJobTemplateFile = new StringBuffer(BaseTest.getResultDir());
-        createDomainJobTemplateFile.append(
-            "/samples/scripts/create-weblogic-domain/domain-home-on-pv/create-domain-job-template.yaml");
-        TestUtils.exec("sed -i -e 's?DYNAMIC?CONFIGURED?g' " + createDomainJobTemplateFile);
-      }
+
+    // domain in image
+    if (domainMap.containsKey("customWdtTemplate")) {
+      TestUtils.copyFile(
+          (String) domainMap.get("customWdtTemplate"),
+          BaseTest.getResultDir()
+              + "/docker-images/OracleWebLogic/samples/12213-domain-home-in-image-wdt/simple-topology.yaml");
+      ExecResult exec =
+          TestUtils.exec(
+              "cat "
+                  + BaseTest.getResultDir()
+                  + "/docker-images/OracleWebLogic/samples/12213-domain-home-in-image-wdt/simple-topology.yaml");
+      logger.log(Level.FINEST, exec.stdout());
+    } else if (clusterType.equalsIgnoreCase("CONFIGURED")) {
+      // domain on pv
+      StringBuffer createDomainJobTemplateFile = new StringBuffer(BaseTest.getResultDir());
+      createDomainJobTemplateFile.append(
+          "/samples/scripts/create-weblogic-domain/domain-home-on-pv/create-domain-job-template.yaml");
+      TestUtils.exec("sed -i -e 's?DYNAMIC?CONFIGURED?g' " + createDomainJobTemplateFile);
     }
   }
 

integration-tests/src/test/resources/multipleclusters/wdtmultipledynclusters.yml

@@ -0,0 +1,41 @@
+domainInfo:
+    AdminUserName: '@@FILE:/u01/oracle/properties/adminuser.properties@@'
+    AdminPassword: '@@FILE:/u01/oracle/properties/adminpass.properties@@'
+topology:
+    Name: '@@PROP:DOMAIN_NAME@@'
+    AdminServerName: '@@PROP:ADMIN_NAME@@'
+    ProductionModeEnabled: '@@PROP:PRODUCTION_MODE_ENABLED@@'
+    Cluster:
+        '@@PROP:CLUSTER_NAME@@':
+            DynamicServers:
+                CalculatedListenPorts: false
+                DynamicClusterSize: 2
+                MaxDynamicClusterSize: '@@PROP:CONFIGURED_MANAGED_SERVER_COUNT@@'
+                ServerNamePrefix: '@@PROP:MANAGED_SERVER_NAME_BASE@@'
+                ServerTemplate: '@@PROP:CLUSTER_NAME@@-template'
+        '@@PROP:CLUSTER_NAME@@-2':
+            DynamicServers:
+                CalculatedListenPorts: false
+                DynamicClusterSize: 2
+                MaxDynamicClusterSize: '@@PROP:CONFIGURED_MANAGED_SERVER_COUNT@@'
+                ServerNamePrefix: '@@PROP:MANAGED_SERVER_NAME_BASE@@-2'
+                ServerTemplate: '@@PROP:CLUSTER_NAME@@-2-template'
+    Server:
+        '@@PROP:ADMIN_NAME@@':
+            ListenPort: '@@PROP:ADMIN_PORT@@'
+            NetworkAccessPoint:
+                T3Channel:
+                    ListenPort: '@@PROP:T3_CHANNEL_PORT@@'
+                    PublicAddress: '@@PROP:T3_PUBLIC_ADDRESS@@'
+                    PublicPort: '@@PROP:T3_CHANNEL_PORT@@'
+    ServerTemplate:
+        '@@PROP:CLUSTER_NAME@@-template':
+            Cluster: '@@PROP:CLUSTER_NAME@@'
+            ListenPort: '@@PROP:MANAGED_SERVER_PORT@@'
+            JTAMigratableTarget:
+                Cluster: '@@PROP:CLUSTER_NAME@@'
+        '@@PROP:CLUSTER_NAME@@-2-template':
+            Cluster: '@@PROP:CLUSTER_NAME@@-2'
+            ListenPort: '@@PROP:MANAGED_SERVER_PORT@@'
+            JTAMigratableTarget:
+                Cluster: '@@PROP:CLUSTER_NAME@@-2'