Integration Test : JTA Service Migration (#2332)

* Integration Test : JTA Service Migration

Co-authored-by: ANTARYAMI.PANIGRAHI@ORACLE.COM <anpanigr@anpanigr-1.subnet1ad2phx.devweblogicphx.oraclevcn.com>
Co-authored-by: ANTARYAMI.PANIGRAHI@ORACLE.COM <anpanigr@anpanigr-2.subnet1ad3phx.devweblogicphx.oraclevcn.com>

Author: 3 people

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiJmsRecovery.java renamed to integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiServiceMigration.java

@@ -71,7 +71,7 @@
  * a stopped pod/managed server to a live pod/managed server.
  * Configuration : 
  *   MII cluster domain with 2 managed servers
- *   Two set of JMS Resources with FileStore and JDBC Store
+ *   Two sets of JMS Resources with FileStore and JDBC Store
  *   All resources are targeted to cluster with enabled JMS service migration
  *   Two Distributed Queue(s) one with FileStore and the other with JDBC Store
  *   Separate ORACLE Datasource for cluster leasing
@@ -82,12 +82,14 @@
  * (d) Make sure all 100 messages got recovered once the 
  *     JMS Service@managed-server2 is migrated to managed-server1 
  * Above steps are repeated for both FileStore and JDBCStore based Distributed Queue.
+ * This test class verifies the JTA Service Migration with shutdown-recovery
+ * migration policy by verifying the JTA Recovery Service runtime MBean
  */
 
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
-@DisplayName("Test JMS service migration on cluster scale down")
+@DisplayName("Test JMS/JTA service migration on cluster scale down")
 @IntegrationTest
-class ItMiiJmsRecovery {
+class ItMiiServiceMigration {
 
   private static String opNamespace = null;
   private static String domainNamespace = null;
@@ -175,7 +177,7 @@ public static void initAll(@Namespaces(2) List<String> namespaces) {
     createSecretForBaseImages(domainNamespace);
 
     // create PV, PVC for logs/data
-    createPV(pvName, domainUid, ItMiiJmsRecovery.class.getSimpleName());
+    createPV(pvName, domainUid, ItMiiServiceMigration.class.getSimpleName());
     createPVC(pvName, pvcName, domainUid, domainNamespace);
 
     // create job to change permissions on PV hostPath
@@ -289,6 +291,52 @@ public void testMiiJmsServiceMigrationWithJdbcStore() {
             "JdbcJmsServer@managed-server2@jms.jdbcUniformQueue");
   }
 
+  /**
+   * Verify JTA Recovery Service is migrated to an available active server.
+   * when a server is shutdown ( cluster is scaled down )
+   * The MigrationPolicy on the ServerTemplate is set to 'shutdown-recovery'
+   * so that the JTA recovery service is migrated to an active server in the
+   * cluster when a server is shutdown. This can be checked by verifying the 
+   * JTARecoveryService runtime MBean for the stopped server in an active 
+   * server. For example say managed server ms2 is down, make sure that the JTA 
+   * recovery service for ms2 is active on the running managed server ms1
+   * Also make sure that the JTA Recovery service (ms2) is migrated back to 
+   * server ms2 when the server ms2 is re-started. 
+   */
+  @Test
+  @Order(3)
+  @DisplayName("Verify JTA Recovery Service migration to an active server")
+  public void testMiiJtaServiceMigration() {
+
+    // Restart the managed server(2) if shutdown by previous test method 
+    // Make sure that JTA Recovery service is active on managed-server2  
+    restartManagedServer("managed-server2");
+    assertTrue(checkJtaRecoveryServiceRuntime("managed-server2", "managed-server2", "true"), 
+         "JTARecoveryService@managed-server2 is not on managed-server2 before migration");
+
+    // Stop the server managed-server2 by patching the cluster
+    boolean psuccess = assertDoesNotThrow(() ->
+            scaleCluster(domainUid, domainNamespace, "cluster-1", 1),
+        String.format("replica patching to 1 failed for domain %s in namespace %s", domainUid, domainNamespace));
+    assertTrue(psuccess,
+        String.format("Cluster replica patching failed for domain %s in namespace %s", domainUid, domainNamespace));
+    checkPodDoesNotExist(managedServerPrefix + "2", domainUid, domainNamespace);
+
+    assertTrue(checkJtaRecoveryServiceRuntime("managed-server1", "managed-server2", "true"), 
+           "JTA RecoveryService@managed-server2 is not migrated to managed-server1");
+    logger.info("JTA RecoveryService@managed-server2 is migrated to managed-server1");
+
+    // Restart the managed server(2) to make sure the JTA Recovery Service is
+    // migrated back to original hosting server 
+    restartManagedServer("managed-server2");
+    assertTrue(checkJtaRecoveryServiceRuntime("managed-server2", "managed-server2", "true"), 
+         "JTARecoveryService@managed-server2 is not on managed-server2 after restart");
+    logger.info("JTA RecoveryService@managed-server2 is migrated back to managed-server1");
+    assertTrue(checkJtaRecoveryServiceRuntime("managed-server1", "managed-server2", "false"), 
+         "JTARecoveryService@managed-server2 is not deactivated on managed-server1 after restart");
+    logger.info("JTA RecoveryService@managed-server2 is deactivated on managed-server1 after restart");
+  }
+
   private void restartManagedServer(String serverName) {
 
     String commonParameters = " -d " + domainUid + " -n " + domainNamespace;
@@ -317,17 +365,18 @@ private void runJmsClientOnAdminPod(String action, String queue) {
   }
 
   /*
-   * Verify the JMS Server Runtime through rest API.
+   * Verify the JMS Server Runtime through REST API.
    * Get the JMSServer Runtime ClusterJmsServer@managed-server2 found on 
    * specified managed server.
-   * @param managedServer name of managed server to look for JMSServerRuntime
-   * @returns true if MBEAN is found otherwise false
+   * @param managedServer name of the managed server to look for JMSServerRuntime
+   * @returns true if MBean is found otherwise false
    **/
   private boolean checkJmsServerRuntime(String managedServer) {
     ExecResult result = null;
     int adminServiceNodePort
         = getServiceNodePort(domainNamespace, getExternalServicePodName(adminServerPodName), "default");
-    StringBuffer curlString = new StringBuffer("status=$(curl --user weblogic:welcome1 ");
+    StringBuffer curlString = new StringBuffer("status=$(curl --user " 
+           + ADMIN_USERNAME_DEFAULT + ":" + ADMIN_PASSWORD_DEFAULT + " ");
     curlString.append("http://" + K8S_NODEPORT_HOST + ":" + adminServiceNodePort)
           .append("/management/weblogic/latest/domainRuntime/serverRuntimes/")
           .append(managedServer)
@@ -351,4 +400,41 @@ private boolean checkJmsServerRuntime(String managedServer) {
     return true;
   }
 
+  /*
+   * Verify the JTA Recovery Service Runtime through REST API.
+   * Get the JTA Recovery Service Runtime for a server on a 
+   * specified managed server.
+   * @param managedServer name of the server to look for RecoveyServerRuntime
+   * @param recoveryService name of RecoveyServerRuntime (managed server) 
+   * @param active is the recovery active (true or false )
+   * @returns true if MBean is found otherwise false
+   **/
+  private boolean checkJtaRecoveryServiceRuntime(String managedServer, String recoveryService, String active) {
+    ExecResult result = null;
+    int adminServiceNodePort
+        = getServiceNodePort(domainNamespace, getExternalServicePodName(adminServerPodName), "default");
+    StringBuffer curlString = new StringBuffer("curl --user " 
+           + ADMIN_USERNAME_DEFAULT + ":" + ADMIN_PASSWORD_DEFAULT + " ");
+    curlString.append("\"http://" + K8S_NODEPORT_HOST + ":" + adminServiceNodePort)
+          .append("/management/weblogic/latest/domainRuntime/serverRuntimes/")
+          .append(managedServer)
+          .append("/JTARuntime/recoveryRuntimeMBeans/")
+          .append(recoveryService)
+          .append("?fields=active&links=none\"")
+          .append(" --show-error ");
+    logger.info("checkJtaRecoveryServiceRuntime: curl command {0}", new String(curlString));
+    withStandardRetryPolicy 
+        .conditionEvaluationListener(
+            condition -> logger.info("Waiting for JTA Recovery Service to migrate "
+                + "(elapsed time {0} ms, remaining time {1} ms)",
+                condition.getElapsedTimeInMS(),
+                condition.getRemainingTimeInMS()))
+        .until(assertDoesNotThrow(() -> {
+          return () -> {
+            return exec(new String(curlString), true).stdout().contains("{\"active\": " + active + "}");
+          };
+        }));
+    return true;
+  }
+
 }

integration-tests/src/test/resources/wdt-models/jms.recovery.yaml

@@ -5,6 +5,14 @@ topology:
         "cluster-1":
            DataSourceForAutomaticMigration:  LeasingDataSource
            AutoMigrationTableName: ACTIVE
+
+    ServerTemplate:
+        "cluster-1-template":
+            Cluster: 'cluster-1'
+            JTAMigratableTarget:
+                Cluster: 'cluster-1'
+                MigrationPolicy: 'shutdown-recovery'
+                StrictOwnershipCheck: true
 resources:
     FileStore:
         ClusterFileStore: