Add test to verify the restarted operator would detect the existing introspector and wait for the job to complete (#2482)

* Add test to verify the restarted operator would detect the existing introspector and wait for the job to complete

Author: maggiehe00

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwMiiDomain.java

@@ -11,6 +11,7 @@
 import io.kubernetes.client.custom.V1Patch;
 import oracle.weblogic.domain.Domain;
 import oracle.weblogic.kubernetes.actions.impl.primitive.Command;
+import oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes;
 import oracle.weblogic.kubernetes.annotations.IntegrationTest;
 import oracle.weblogic.kubernetes.annotations.Namespaces;
 import oracle.weblogic.kubernetes.logging.LoggingFacade;
@@ -33,17 +34,21 @@
 import static oracle.weblogic.kubernetes.TestConstants.OCIR_SECRET_NAME;
 import static oracle.weblogic.kubernetes.actions.ActionConstants.MODEL_DIR;
 import static oracle.weblogic.kubernetes.actions.ActionConstants.RESOURCE_DIR;
+import static oracle.weblogic.kubernetes.actions.TestActions.getCurrentIntrospectVersion;
 import static oracle.weblogic.kubernetes.actions.TestActions.patchDomainCustomResource;
 import static oracle.weblogic.kubernetes.actions.impl.primitive.Command.defaultCommandParams;
 import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.verifyUpdateWebLogicCredential;
+import static oracle.weblogic.kubernetes.utils.CommonOperatorUtils.restartOperator;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodDeleted;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodExists;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createMiiImageAndVerify;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createOcirRepoSecret;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createOpsswalletpasswordSecret;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createRcuAccessSecret;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.dockerLoginAndPushImageToRegistry;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getIntrospectJobName;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.patchServerStartPolicy;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.updateRcuAccessSecret;
@@ -54,6 +59,7 @@
 import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
 import static org.awaitility.Awaitility.with;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -140,6 +146,10 @@ RCUSCHEMAPREFIX, dbNamespace, getNextFreePort(), dbUrl),
 
   /**
    * Create a basic FMW model in image domain.
+   * Create the FMW domain with introspectVersion
+   * After domain is created and introspector exists restart the operator.
+   * Verify the restarted operator can find the existing introspector and wait for it to complete
+   * rather than replacing a new introspector.
    * Verify Pod is ready and service exists for both admin server and managed servers.
    * Verify EM console is accessible.
    */
@@ -213,6 +223,31 @@ public void testFmwModelInImage() {
         fmwMiiImage);
 
     createDomainAndVerify(domain, fmwDomainNamespace);
+
+    //verify the introspector pod is created and runs
+    logger.info("Verifying introspector pod is created, runs");
+    String introspectPodNameBase = getIntrospectJobName(domainUid);
+    logger.info("Checking introspector pod exists and introspect version");
+    checkPodExists(introspectPodNameBase, domainUid, fmwDomainNamespace);
+    String introspectPodName = assertDoesNotThrow(() -> Kubernetes.listPods(fmwDomainNamespace, null)
+        .getItems().get(0).getMetadata().getName(),
+        String.format("Get intrspector pod name failed with ApiException in namespace %s", fmwDomainNamespace));
+    String introspectVersion1 =
+        assertDoesNotThrow(() -> getCurrentIntrospectVersion(domainUid, fmwDomainNamespace));
+    logger.info("Before restarting operator introspector pod name is: {0}, introspectVersion is: {1}",
+        introspectPodName, introspectVersion1);
+
+    logger.info("Restarting operator in the namespace: " + opNamespace);
+    restartOperator(opNamespace);
+    //verify the exact same introspector pod exists and Version does not change
+    logger.info("Checking the exact same introspector pod {0} exists in the namespace {1}",
+        introspectPodName, fmwDomainNamespace);
+    checkPodExists(introspectPodName, domainUid, fmwDomainNamespace);
+    String introspectVersion2 =
+        assertDoesNotThrow(() -> getCurrentIntrospectVersion(domainUid, fmwDomainNamespace));
+    logger.info("After operator restart introspectVersion is: " + introspectVersion2);
+    assertEquals(introspectVersion1, introspectVersion2, "introspectVersion changes after operator restart");
+
     verifyDomainReady(fmwDomainNamespace, domainUid, replicaCount);
   }
 
@@ -358,4 +393,5 @@ private boolean patchDomainWithWalletFileSecret(String opssWalletFileSecretName)
 
     return patchDomainCustomResource(domainUid, fmwDomainNamespace, patch, V1Patch.PATCH_FORMAT_JSON_PATCH);
   }
+  
 }

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonOperatorUtils.java

@@ -0,0 +1,72 @@
+// Copyright (c) 2021, Oracle and/or its affiliates.
+// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+package oracle.weblogic.kubernetes.utils;
+
+import oracle.weblogic.kubernetes.logging.LoggingFacade;
+import org.awaitility.core.ConditionFactory;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static oracle.weblogic.kubernetes.TestConstants.OPERATOR_RELEASE_NAME;
+import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorPodName;
+import static oracle.weblogic.kubernetes.actions.TestActions.startOperator;
+import static oracle.weblogic.kubernetes.actions.TestActions.stopOperator;
+import static oracle.weblogic.kubernetes.assertions.TestAssertions.operatorIsReady;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodDoesNotExist;
+import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
+import static org.awaitility.Awaitility.with;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/**
+ * The common utility class for Operator.
+ */
+public class CommonOperatorUtils {
+
+  private static final ConditionFactory withStandardRetryPolicy
+      = with().pollDelay(2, SECONDS)
+      .and().with().pollInterval(10, SECONDS)
+      .atMost(5, MINUTES).await();
+
+
+  /**
+   * Restart Operator by changing replica to 0 in operator deployment to stop Operator
+   * and changing replica back to 1 to start Operator.
+   * @param opNamespace namespace where Operator exists
+   */
+  public static void restartOperator(String opNamespace) {
+    LoggingFacade logger = getLogger();
+    // get operator pod name
+    String operatorPodName = assertDoesNotThrow(
+        () -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace));
+    assertNotNull(operatorPodName, "Operator pod name returned is null");
+    logger.info("Operator pod name {0}", operatorPodName);
+
+    // stop operator by changing replica to 0 in operator deployment
+    assertTrue(stopOperator(opNamespace), "Couldn't stop the Operator");
+
+    // check operator pod is not running
+    checkPodDoesNotExist(operatorPodName, null, opNamespace);
+
+    // start operator by changing replica to 1 in operator deployment
+    assertTrue(startOperator(opNamespace), "Couldn't start the Operator");
+
+    // check operator is running
+    logger.info("Check Operator pod is running in namespace {0}", opNamespace);
+    withStandardRetryPolicy
+        .conditionEvaluationListener(
+            condition -> logger.info("Waiting for operator to be running in namespace {0} "
+                    + "(elapsed time {1}ms, remaining time {2}ms)",
+                opNamespace,
+                condition.getElapsedTimeInMS(),
+                condition.getRemainingTimeInMS()))
+        .until(operatorIsReady(opNamespace));
+
+    logger.info("Operator pod is restarted in namespace {0}", opNamespace);
+
+  }
+
+}

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/FmwUtils.java

@@ -75,6 +75,7 @@ public static Domain createDomainResource(
                 .namespace(domNamespace))
             .includeServerOutInPodLog(true)
             .serverStartPolicy("IF_NEEDED")
+            .introspectVersion("1")
             .serverPod(new ServerPod()
                 .addEnvItem(new V1EnvVar()
                     .name("JAVA_OPTIONS")