added usecases for shutdown props

marinakog · marinakog · commit c25654b869ea · 2019-06-07T15:42:42.000-07:00
diff --git a/integration-tests/USECASES.MD b/integration-tests/USECASES.MD
@@ -79,6 +79,17 @@ Basic Use Cases described above are verified in all the domain configurations. A
 | Server pods restarted by changing restartVersion at cluster level | Verify managed server pods part of the dynamic cluster are restarted by adding restartVersion property at cluster level : `restartVersion: "v1.1"` |
 | Server pods restarted by changing restartVersion at domain level | Verify all the server pods in the weblogic domain are restarted by adding restartVersion property at domain level: `restartVersion: "v1.1"` |
 
+| Server Pods Shutdown options| Use Case |
+| --- | --- |
+| Shutdown options: domain level| Verify that shutdown options set at the domain level will apply to all servers in the domain  |
+| Shutdown options: managed server level | Verify that shutdown options set at the managed server level will apply only to the specified server  |
+| Shutdown options: managed server level | Verify that shutdown options set at the cluster level will apply only to the all servers in that cluster  |
+| Shutdown options: use env variables | Verify that shutdown options set using env vars will apply   |
+| Shutdown option ignoreSessions | Verify that shutdown option: ignoreSessions(true, false) will effect shutdown time of the pod   |
+| Shutdown option timeout | Verify that shutdown option: timeout will effect shutdown time of the pod   |
+| Shutdown option Forced | Verify that shutdown option: Forced will effect shutdown time of the pod   |
+| Shutdown overrides | Verify that shutdown options set on (managed server or cluster) (lowest level) will take priority over domain level settings   |
+
 Configuration Overrides Usecases
 
 | Override | Usecase |
diff --git a/integration-tests/src/test/java/oracle/kubernetes/operator/ITPodsShutdown.java b/integration-tests/src/test/java/oracle/kubernetes/operator/ITPodsShutdown.java
@@ -18,7 +18,6 @@
 import oracle.kubernetes.operator.utils.ExecResult;
 import oracle.kubernetes.operator.utils.Operator;
 import oracle.kubernetes.operator.utils.TestUtils;
-import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.BeforeClass;
@@ -82,11 +81,7 @@ public static void staticPrepare() throws Exception {
       // Wait some time for deployment gets ready
       // Thread.sleep(10 * 1000);
       // restart managed server to get default shutdown time with default shutdown options
-      getDefaultShutdownTime();
 
-      logger.info(
-          "termination time for managed-server1 with default shutdown options is:"
-              + terminationDefaultOptionsTime);
     }
   }
 
@@ -95,7 +90,7 @@ public static void staticPrepare() throws Exception {
    *
    * @throws Exception
    */
-  @AfterClass
+  // @AfterClass
   public static void staticUnPrepare() throws Exception {
     if (!QUICKTEST) {
       logger.info("+++++++++++++++++++++++++++++++++---------------------------------+");
@@ -127,11 +122,14 @@ public void testAddShutdownOptionsToMS() throws Exception {
     DomainCRD crd = new DomainCRD(originalYaml);
 
     Map<String, Object> shutdownProps = new HashMap();
-    shutdownProps.put("timeoutSeconds", 60);
+    shutdownProps.put("timeoutSeconds", 160);
     shutdownProps.put("shutdownType", "Forced");
     shutdownProps.put("ignoreSessions", true);
     crd.addShutDownOptionToMS("managed-server1", shutdownProps);
-    updateCRDYamlVerifyShutdown(crd, 1000);
+    updateCRDYamlVerifyShutdown(crd, 0);
+    Assert.assertTrue(checkShutdownUpdatedProp(domainUid + "-admin-server", "Graceful"));
+    Assert.assertTrue(
+        checkShutdownUpdatedProp(domainUid + "-managed-server1", "Forced", "160", "true"));
   }
 
   /**
@@ -146,8 +144,8 @@ public void testAddShutdownOptionToCluster() throws Exception {
     String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
     logTestBegin(testMethodName);
     resetDomainCRD();
-    String podName = domainUid + "-managed-server1";
     Files.createDirectories(Paths.get(shutdownTmpDir));
+
     // Modify the original domain yaml to include shutdown options in cluster-1 node
     DomainCRD crd = new DomainCRD(originalYaml);
 
@@ -158,6 +156,8 @@ public void testAddShutdownOptionToCluster() throws Exception {
 
     crd.addShutdownOptionsToCluster(domain.getClusterName(), shutdownProps);
     updateCRDYamlVerifyShutdown(crd, 0);
+    Assert.assertTrue(checkShutdownUpdatedProp(domainUid + "-admin-server", "Graceful"));
+    Assert.assertTrue(checkShutdownUpdatedProp(domainUid + "-managed-server1", "Forced"));
     logger.log(Level.INFO, "SUCCESS - {0}", testMethodName);
   }
 
@@ -173,19 +173,46 @@ public void testAddShutdownOptionsToDomain() throws Exception {
     String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
     logTestBegin(testMethodName);
     resetDomainCRD();
-    String podName = domainUid + "-managed-server1";
     Files.createDirectories(Paths.get(shutdownTmpDir));
     // Modify the original domain yaml to include shutdown options in domain spec node
     DomainCRD crd = new DomainCRD(originalYaml);
 
     Map<String, Object> shutdownProps = new HashMap();
     shutdownProps.put("timeoutSeconds", 160);
-    shutdownProps.put("ignoreSessions", false);
     crd.addShutdownOptionToDomain(shutdownProps);
+
+    updateCRDYamlVerifyShutdown(crd, 0);
+    Assert.assertTrue(checkShutdownUpdatedProp(domainUid + "-admin-server", "160"));
+    Assert.assertTrue(checkShutdownUpdatedProp(domainUid + "-managed-server1", "160"));
+
+    logger.log(Level.INFO, "SUCCESS - {0}", testMethodName);
+  }
+
+  /**
+   * Add shutdown option IgnoreSessions at managed server level and verify all pods are Terminated
+   * according to the setting
+   *
+   * @throws Exception when domain.yaml cannot be read or modified
+   */
+  @Test
+  public void testAddShutdownOptionsToMSIgnoreSessions() throws Exception {
+
+    Assume.assumeFalse(QUICKTEST);
+    String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
+    logTestBegin(testMethodName);
+
+    resetDomainCRD();
+    Files.createDirectories(Paths.get(shutdownTmpDir));
+    // Modify the original domain yaml to include shutdown options in domain spec node
+    DomainCRD crd = new DomainCRD(originalYaml);
+
+    Map<String, Object> shutdownProps = new HashMap();
+    shutdownProps.put("timeoutSeconds", 160);
+    shutdownProps.put("ignoreSessions", false);
+    crd.addShutDownOptionToMS("managed-server1", shutdownProps);
     long delayTime = 50 * 1000;
     updateCRDYamlVerifyShutdown(crd, delayTime);
-    Assert.assertTrue(
-        checkShutdownUpdatedProp(domainUid + "-admin-server", "160", "false", "Graceful"));
+
     Assert.assertTrue(
         checkShutdownUpdatedProp(domainUid + "-managed-server1", "160", "false", "Graceful"));
     if (terminationTime < delayTime) {
@@ -195,15 +222,13 @@ public void testAddShutdownOptionsToDomain() throws Exception {
     long terminationTimeWithIgnoreSessionFalse = terminationTime;
     logger.info(
         " Termination time with ignoreSession=false :" + terminationTimeWithIgnoreSessionFalse);
-    resetDomainCRD();
+
     shutdownProps = new HashMap();
     shutdownProps.put("timeoutSeconds", 160);
     shutdownProps.put("ignoreSessions", true);
-    crd.addShutdownOptionToDomain(shutdownProps);
 
+    crd.addShutDownOptionToMS("managed-server1", shutdownProps);
     updateCRDYamlVerifyShutdown(crd, delayTime);
-    Assert.assertTrue(
-        checkShutdownUpdatedProp(domainUid + "-admin-server", "160", "true", "Graceful"));
     Assert.assertTrue(
         checkShutdownUpdatedProp(domainUid + "-managed-server1", "160", "true", "Graceful"));
 
@@ -215,30 +240,73 @@ public void testAddShutdownOptionsToDomain() throws Exception {
       logger.info("FAILURE: did not ignore opened sessions during shutdown");
       throw new Exception("FAILURE: did not ignore opened sessions during shutdown");
     }
+    logger.log(Level.INFO, "SUCCESS - {0}", testMethodName);
+  }
+
+  /**
+   * Add shutdown option Timeout at managed server level and verify all pods are Terminated
+   * according to the setting
+   *
+   * @throws Exception when domain.yaml cannot be read or modified
+   */
+  @Test
+  public void testAddShutdownOptionsToMSTimeout() throws Exception {
+
+    Assume.assumeFalse(QUICKTEST);
+    String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
+    logTestBegin(testMethodName);
+
     resetDomainCRD();
+    Files.createDirectories(Paths.get(shutdownTmpDir));
+    // Modify the original domain yaml to include shutdown options in domain spec node
+    DomainCRD crd = new DomainCRD(originalYaml);
+
+    long delayTime = 50 * 1000;
     // testing timeout
-    shutdownProps = new HashMap();
+    Map<String, Object> shutdownProps = new HashMap();
     shutdownProps.put("timeoutSeconds", 20);
     shutdownProps.put("ignoreSessions", false);
-    crd.addShutdownOptionToDomain(shutdownProps);
+    crd.addShutDownOptionToMS("managed-server1", shutdownProps);
 
     updateCRDYamlVerifyShutdown(crd, delayTime);
-    Assert.assertTrue(
-        checkShutdownUpdatedProp(domainUid + "-admin-server", "20", "false", "Graceful"));
+
     Assert.assertTrue(
         checkShutdownUpdatedProp(domainUid + "-managed-server1", "20", "false", "Graceful"));
     if (terminationTime > (3 * 20 * 1000)) {
       logger.info("\"FAILURE: ignored timeoutValue during shutdown");
       throw new Exception("FAILURE: ignored timeoutValue during shutdown");
     }
-    // testing Forced
+    logger.log(Level.INFO, "SUCCESS - {0}", testMethodName);
+  }
+
+  /**
+   * Add shutdown option Forced at managed server level and verify all pods are Terminated according
+   * to the setting
+   *
+   * @throws Exception when domain.yaml cannot be read or modified
+   */
+  @Test
+  public void testAddShutdownOptionsToMSForced() throws Exception {
+
+    Assume.assumeFalse(QUICKTEST);
+    String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
+    logTestBegin(testMethodName);
+
     resetDomainCRD();
-    shutdownProps = new HashMap();
-    shutdownProps.put("shutdownType", "Forced");
-    crd.addShutdownOptionToDomain(shutdownProps);
+    if (terminationDefaultOptionsTime == 0) {
+      getDefaultShutdownTime();
+    }
+    Files.createDirectories(Paths.get(shutdownTmpDir));
+    // Modify the original domain yaml to include shutdown options in domain spec node
+    DomainCRD crd = new DomainCRD(originalYaml);
 
+    long delayTime = 50 * 1000;
+    // testing timeout
+    Map<String, Object> shutdownProps = new HashMap();
+    shutdownProps.put("shutdownType", "Forced");
+    crd.addShutDownOptionToMS("managed-server1", shutdownProps);
     updateCRDYamlVerifyShutdown(crd, delayTime);
-    Assert.assertTrue(checkShutdownUpdatedProp(domainUid + "-admin-server", "Forced"));
+
     Assert.assertTrue(checkShutdownUpdatedProp(domainUid + "-managed-server1", "Forced"));
     if ((2 * terminationDefaultOptionsTime < terminationTime)) {
       logger.info("\"FAILURE: ignored timeout Forced value during shutdown");
@@ -258,7 +326,6 @@ public void testAddEnvShutdownOptions() throws Exception {
     String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
     logTestBegin(testMethodName);
     resetDomainCRD();
-    String podName = domainUid + "-managed-server1";
     Files.createDirectories(Paths.get(shutdownTmpDir));
     // Modify the original domain yaml to include shutdown env vars options in domain spec node
     DomainCRD crd = new DomainCRD(originalYaml);
@@ -284,7 +351,7 @@ public void testShutdownOptionsOverrideViaEnv() throws Exception {
     String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
     logTestBegin(testMethodName);
     resetDomainCRD();
-    String podName = domainUid + "-managed-server1";
+
     Files.createDirectories(Paths.get(shutdownTmpDir));
     // Modify the original domain yaml to include shutdown options in domain spec node
     DomainCRD crd = new DomainCRD(originalYaml);
@@ -320,7 +387,6 @@ public void testShutdownOptionsOverrideClusterLevel() throws Exception {
     String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
     logTestBegin(testMethodName);
     resetDomainCRD();
-    String podName = domainUid + "-managed-server1";
     Files.createDirectories(Paths.get(shutdownTmpDir));
     // Modify the original domain yaml to include shutdown env vars options in domain spec node
     DomainCRD crd = new DomainCRD(originalYaml);
@@ -370,13 +436,14 @@ private static void updateCRDYamlVerifyShutdown(DomainCRD crd, long delayTime) t
     Files.write(path, modYaml.getBytes(charset));
     modifiedYaml = path.toString();
     // Apply the new yaml to update the domain crd
+
     logger.log(Level.INFO, "kubectl delete -f {0}", originalYaml);
     ExecResult exec = TestUtils.exec("kubectl delete -f " + originalYaml);
     logger.info(exec.stdout());
     TestUtils.checkPodDeleted(domainUid + "-managed-server1", domainNS);
     TestUtils.checkPodDeleted(domainUid + "-admin-server", domainNS);
 
-    logger.log(Level.INFO, "kubectl apply -f ", path.toString());
+    logger.log(Level.INFO, "kubectl apply -f {0}", path.toString());
     exec = TestUtils.exec("kubectl apply -f " + path.toString());
     logger.info(exec.stdout());
 
@@ -396,11 +463,15 @@ private static void updateCRDYamlVerifyShutdown(DomainCRD crd, long delayTime) t
 
     logger.info("Checking termination time");
     terminationTime = checkShutdownTime(domainUid + "-managed-server1");
+    logger.info(" termination time " + terminationTime);
   }
 
   private static void getDefaultShutdownTime() throws Exception {
     shutdownServer("managed-server1");
     terminationDefaultOptionsTime = checkShutdownTime(domainUid + "-managed-server1");
+    logger.info(
+        " termination pod's time with default shutdown options is: "
+            + terminationDefaultOptionsTime);
   }
 
   private static void resetDomainCRD() throws Exception {
@@ -440,6 +511,7 @@ private static void destroyDomain() throws Exception {
    *
    * @param testAppPath - URL path for webapp
    * @param domain - Domain where webapp deployed
+   * @param deployApp - option to build and deployApp
    * @throws Exception
    */
   public static void callWebApp(String testAppPath, Domain domain, boolean deployApp)
@@ -448,6 +520,7 @@ public static void callWebApp(String testAppPath, Domain domain, boolean deployA
     if (deployApp) {
       domain.buildDeployJavaAppInPod(
           testAppName, scriptName, BaseTest.getUsername(), BaseTest.getPassword());
+      domain.callWebAppAndVerifyLoadBalancing(testAppName, false);
     }
     String nodePortHost = domain.getHostNameForCurl();
     int nodePort = domain.getLoadBalancerWebPort();
@@ -492,6 +565,7 @@ private void scaleCluster(int replicas) throws Exception {
    * @param podName
    * @throws Exception
    */
+  /*
   public static long checkShutdownTime(String podName) throws Exception {
     String namespace = domainNS;
     long startTime = System.currentTimeMillis();
@@ -507,6 +581,7 @@ public static long checkShutdownTime(String podName) throws Exception {
     logger.info("The time to terminate the pod is " + (currentTime - startTime));
     return (currentTime - startTime);
   }
+  */
 
   /**
    * shutdown managed server
@@ -554,6 +629,51 @@ public static boolean checkShutdownUpdatedProp(String podName, String... props)
     if (props.length == propFound.size()) found = true;
     return found;
   }
+
+  public static long checkShutdownTime(String podName) throws Exception {
+    long startTime = System.currentTimeMillis();
+    long endTime = 0;
+    int maxIterations = 50;
+    int waitPodTime = 5;
+    String matchStr = "Terminating";
+    StringBuffer cmd = new StringBuffer();
+    cmd.append("kubectl get pod ").append(podName).append(" -n ").append(domainNS);
+    int i = 0;
+    while (i < maxIterations) {
+      ExecResult result = ExecCommand.exec(cmd.toString());
+
+      // pod might not have been created or if created loop till condition
+      if ((result.exitValue() == 0 && result.stdout().contains(matchStr))) {
+        logger.info("Output for " + cmd + "\n" + result.stdout() + "\n " + result.stderr());
+
+        // check for last iteration
+        if (i == (maxIterations - 1)) {
+          throw new RuntimeException(
+              "FAILURE: pod " + podName + " is still in " + matchStr + " status, exiting!");
+        }
+        logger.info(
+            "Pod "
+                + podName
+                + " is "
+                + matchStr
+                + " Ite ["
+                + i
+                + "/"
+                + maxIterations
+                + "], sleeping "
+                + waitPodTime
+                + " seconds more");
+
+        Thread.sleep(maxIterations * 1000);
+        i++;
+      } else {
+        endTime = System.currentTimeMillis();
+        logger.info("Pod " + podName + " is not in the " + matchStr + " status or does not exists");
+        break;
+      }
+    }
+    return (endTime - startTime);
+  }
 }
 
 class SessionDelayThread implements Runnable {
