Added tests to use script scaleCluster.sh/ClusterStatus.sh (#2350)

* Added tests to use script scaleCluster.sh/ClusterStatus.sh

* Added more desriptions

Co-authored-by: huizhao <huizhao@huizhao-1.subnet2ad1phx.devweblogicphx.oraclevcn.com>
Co-authored-by: huizhao <huizhao@phx3187a26.subnet4ad3phx.devweblogicphx.oraclevcn.com>

Author: 3 people

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItServerStartPolicy.java

@@ -64,6 +64,7 @@
 import static oracle.weblogic.kubernetes.utils.CommonPatchTestUtils.patchServerStartPolicy;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkClusterReplicaCountMatches;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodDeleted;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodDoesNotExist;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodInitializing;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createConfigMapAndVerify;
@@ -89,7 +90,6 @@
  * managed server. The replica count is set to 1 and serverStartPolicy is set 
  * to IF_NEEDED at managed server level. 
  */
-
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
 @DisplayName("ServerStartPolicy attribute in different levels in a MII domain")
 @IntegrationTest
@@ -104,6 +104,8 @@ class ItServerStartPolicy {
   public static final String START_CLUSTER_SCRIPT = "startCluster.sh";
   public static final String STOP_DOMAIN_SCRIPT = "stopDomain.sh";
   public static final String START_DOMAIN_SCRIPT = "startDomain.sh";
+  public static final String SCALE_CLUSTER_SCRIPT = "scaleCluster.sh";
+  public static final String STATUS_CLUSTER_SCRIPT = "clusterStatus.sh";
   public static final String managedServerNamePrefix = "managed-server";
   public static final String CLUSTER_1 = "cluster-1";
   public static final String CLUSTER_2 = "cluster-2";
@@ -1130,6 +1132,131 @@ public void testRestartingMSWithExplicitServerStartStateWhileVaryingReplicaCount
     logger.info("managed server " + serverName + " restarted successfully.");
   }
 
+  /**
+   * Scale the configured cluster using the sample script scaleCluster.sh script
+   * Verify that server(s) in the configured cluster are scaled up and in RUNNING state.
+   * Verify that server(s) in the dynamic cluster are not affected.
+   * Restore the env using the sample script stopServer.sh.
+   */
+  @Order(19)
+  @Test
+  @DisplayName("Scale the configured cluster with scaleCluster.sh script")
+  public void testConfigClusterScale() {
+    int newReplicaCount = 2;
+    String configServerName = "config-cluster-server" + newReplicaCount;
+    String configServerPodName = domainUid + "-" + configServerName;
+    String dynamicServerPodName = domainUid + "-managed-server" + newReplicaCount;
+
+    // use clusterStatus.sh to make sure the server-to-be-test doesn't exist
+    // String regex matches below
+    // cluster        min  max  goal  current  ready
+    // clusterName     1    5    1      1       1
+    String regex = ".*" + CLUSTER_1 + "(\\s+)1(\\s+)5(\\s+)1(\\s+)1(\\s+)1";
+    scalingClusters(CLUSTER_1, dynamicServerPodName, replicaCount, regex, false);
+    // String regex matches below
+    // cluster        min  max  goal  current  ready
+    // clusterName     0    2    1      1       1
+    regex = ".*" + CLUSTER_2 + "(\\s+)0(\\s+)2(\\s+)1(\\s+)1(\\s+)1";
+    scalingClusters(CLUSTER_2, configServerPodName, replicaCount, regex, false);
+
+    // use scaleCluster.sh to scale a dynamic cluster and
+    // use clusterStatus.sh to verify scaling results
+    // String regex matches below
+    // cluster        min  max  goal  current  ready
+    // clusterName     0    2    2       2      2
+    regex = ".*" + CLUSTER_2 + "(\\s+)0(\\s+)2(\\s+)2(\\s+)2(\\s+)2";
+    scalingClusters(CLUSTER_2, configServerPodName, newReplicaCount, regex, true);
+
+    // check managed server from other cluster are not affected
+    logger.info("Check dynamic managed server pods are not affected");
+    assertDoesNotThrow(() -> assertTrue(checkClusterReplicaCountMatches(CLUSTER_1,
+        domainUid, domainNamespace, replicaCount)));
+    checkPodDoesNotExist(dynamicServerPodName, domainUid, domainNamespace);
+
+    // use clusterStatus.sh to restore test env
+    // String regex matches below
+    // cluster        min  max  goal  current  ready
+    // clusterName     0    2    1      1       1
+    regex = ".*" + CLUSTER_2 + "(\\s+)0(\\s+)2(\\s+)1(\\s+)1(\\s+)1";
+    scalingClusters(CLUSTER_2, configServerPodName, replicaCount, regex, false);
+  }
+
+  /**
+   * Scale the dynamic cluster using the sample script scaleCluster.sh script
+   * Verify that server(s) in the dynamic cluster are scaled up and in RUNNING state.
+   * Verify that server(s) in the configured cluster are not affected.
+   * Restore the env using the sample script stopServer.sh.
+   */
+  @Order(20)
+  @Test
+  @DisplayName("Scale the dynamic cluster with scaleCluster.sh script")
+  public void testDynamicClusterScale() {
+    int newReplicaCount = 2;
+    String dynamicServerName = "managed-server" + newReplicaCount;
+    String dynamicServerPodName = domainUid + "-" + dynamicServerName;
+    String configServerPodName = domainUid + "-config-cluster-server" + newReplicaCount;
+
+    // use clusterStatus.sh to make sure the server-to-be-test doesn't exist
+    // String regex matches below
+    // cluster        min  max  goal  current  ready
+    // clusterName     1    5    1     1      1
+    String regex = ".*" + CLUSTER_1 + "(\\s+)1(\\s+)5(\\s+)1(\\s+)1(\\s+)1";
+    scalingClusters(CLUSTER_1, dynamicServerPodName, replicaCount, regex, false);
+    // String regex matches below
+    // cluster        min  max  goal  current  ready
+    // clusterName     0    2    1     1      1
+    regex = ".*" + CLUSTER_2 + "(\\s+)0(\\s+)2(\\s+)1(\\s+)1(\\s+)1";
+    scalingClusters(CLUSTER_2, configServerPodName, replicaCount, regex, false);
+
+    // use scaleCluster.sh to scale a dynamic cluster and
+    // use clusterStatus.sh to verify scaling results
+    // String regex matches below
+    // cluster        min  max  goal  current  ready
+    // clusterName     1    5    2       2      2
+    regex = ".*" + CLUSTER_1 + "(\\s+)1(\\s+)5(\\s+)2(\\s+)2(\\s+)2";
+    scalingClusters(CLUSTER_1, dynamicServerPodName, newReplicaCount, regex, true);
+
+    // check managed server from other cluster are not affected
+    logger.info("Check configured managed server pods are not affected");
+    assertDoesNotThrow(() -> assertTrue(checkClusterReplicaCountMatches(CLUSTER_2,
+        domainUid, domainNamespace, replicaCount)));
+    checkPodDoesNotExist(configServerPodName, domainUid, domainNamespace);
+
+    // use clusterStatus.sh to restore test env
+    // String regex matches below
+    // cluster        min  max  goal  current  ready
+    // clusterName     1    5    1     1      1
+    regex = ".*" + CLUSTER_1 + "(\\s+)1(\\s+)5(\\s+)1(\\s+)1(\\s+)1";
+    scalingClusters(CLUSTER_1, dynamicServerPodName, replicaCount, regex, false);
+  }
+
+  private void scalingClusters(String clusterName, String serverPodName, int replicaNum,
+                               String regex, boolean checkPodExist) {
+    // use scaleCluster.sh to scale a given cluster
+    logger.info("Scale cluster {0} using the script scaleCluster.sh", clusterName);
+    String result =  assertDoesNotThrow(() ->
+        executeLifecycleScript(SCALE_CLUSTER_SCRIPT, CLUSTER_LIFECYCLE, clusterName, " -r " + replicaNum, false),
+        String.format("Failed to run %s", SCALE_CLUSTER_SCRIPT));
+
+    if (checkPodExist) {
+      checkPodReadyAndServiceExists(serverPodName, domainUid, domainNamespace);
+    } else {
+      checkPodDoesNotExist(serverPodName, domainUid, domainNamespace);
+    }
+
+    // verify that scaleCluster.sh does scale to a required replica number
+    assertDoesNotThrow(() -> assertTrue(checkClusterReplicaCountMatches(clusterName,
+        domainUid, domainNamespace, replicaNum)));
+
+    // use clusterStatus.sh to verify scaling results
+    result =  assertDoesNotThrow(() ->
+        executeLifecycleScript(STATUS_CLUSTER_SCRIPT, CLUSTER_LIFECYCLE, clusterName),
+        String.format("Failed to run %s", STATUS_CLUSTER_SCRIPT));
+
+    assertTrue(verifyExecuteResult(result, regex), "The script should scale the given cluster: " + clusterName);
+    logger.info("The cluster {0} scaled successfully.", clusterName);
+  }
+
   private static void createDomainSecret(String secretName, String username, String password, String domNamespace) {
     Map<String, String> secretMap = new HashMap<>();
     secretMap.put("username", username);
@@ -1284,6 +1411,7 @@ private String executeLifecycleScript(String script,
                                         boolean checkResult,
                                         String... args) {
     String domainName = (args.length == 0) ? domainUid : args[0];
+
     CommandParams params;
     String commonParameters = " -d " + domainName + " -n " + domainNamespace;
     params = new CommandParams().defaults();
@@ -1292,6 +1420,10 @@ private String executeLifecycleScript(String script,
           + Paths.get(domainLifecycleSamplePath.toString(), "/" + script).toString()
           + commonParameters + " -s " + entityName + " " + extraParams);
     } else if (scriptType.equals(CLUSTER_LIFECYCLE)) {
+      if (extraParams.contains("-r")) {
+        commonParameters += " " + extraParams;
+      }
+
       params.command("sh "
           + Paths.get(domainLifecycleSamplePath.toString(), "/" + script).toString()
           + commonParameters + " -c " + entityName);