Merge pull request #1071 from oracle/fix-int-test-external-jenkins

Fix integration tests to run on remote multi node k8s cluster(external jenkins)

Author: rjeberhard

integration-tests/README.md

@@ -228,8 +228,20 @@ SHARED_CLUSTER=true:
 | IMAGE_NAME_OPERATOR | Docker image name for operator. Default is weblogic-kubernetes-operator |
 | IMAGE_PULL_POLICY_OPERATOR | Default 'Never'. |
 | IMAGE_PULL_SECRET_OPERATOR | Default ''. |
- | IMAGE_PULL_SECRET_WEBLOGIC | Default ''.
+| IMAGE_PULL_SECRET_WEBLOGIC | Default ''. |
 
+The below env variables are required for SHARED_CLUSTER=true:
+
+| Variable | Description |
+| --- | --- |
+| REPO_REGISTRY | OCIR Server to push/pull the Operator image |
+| REPO_USERNAME | OCIR Username |
+| REPO_PASSWORD | OCIR token |
+| REPO_EMAIL | OCIR email |
+| DOCKER_USERNAME | Docker username to pull the Weblogic image  |
+| DOCKER_PASSWORD | Docker password |
+| DOCKER_EMAIL | Docker email |
+| K8S_NODEPORT_HOST | DNS name of a Kubernetes worker node. |
 
 Successful run will have the output like below:
 ```

integration-tests/src/test/java/oracle/kubernetes/operator/BaseTest.java

@@ -52,6 +52,7 @@ public class BaseTest {
   public static boolean QUICKTEST;
   public static boolean SMOKETEST;
   public static boolean JENKINS;
+  public static boolean SHARED_CLUSTER;
   public static boolean INGRESSPERDOMAIN = true;
 
   private static String resultRoot = "";
@@ -83,6 +84,9 @@ public class BaseTest {
     if (System.getenv("JENKINS") != null) {
       JENKINS = new Boolean(System.getenv("JENKINS")).booleanValue();
     }
+    if (System.getenv("SHARED_CLUSTER") != null) {
+      SHARED_CLUSTER = new Boolean(System.getenv("SHARED_CLUSTER")).booleanValue();
+    }
     if (System.getenv("INGRESSPERDOMAIN") != null) {
       INGRESSPERDOMAIN = new Boolean(System.getenv("INGRESSPERDOMAIN")).booleanValue();
     }
@@ -128,7 +132,7 @@ public static void initialize(String appPropsFile) throws Exception {
     }
 
     // for manual/local run, do cleanup
-    if (System.getenv("SHARED_CLUSTER") == null && System.getenv("JENKINS") == null) {
+    if (!JENKINS) {
 
       // delete k8s artifacts created if any, delete PV directories
       ExecResult clnResult = cleanup();
@@ -143,19 +147,25 @@ public static void initialize(String appPropsFile) throws Exception {
               + clnResult.stderr());
     }
 
-    if (System.getenv("JENKINS") != null) {
-      logger.info("Creating " + resultRoot + "/acceptance_test_tmp");
+    if (JENKINS) {
+      logger.info("Deleting and creating " + resultRoot + "/acceptance_test_tmp");
       TestUtils.exec(
-          "/usr/local/packages/aime/ias/run_as_root \"mkdir -p "
+          "/usr/local/packages/aime/ias/run_as_root \"rm -rf "
+              + resultRoot
+              + "/acceptance_test_tmp\" && "
+              + "/usr/local/packages/aime/ias/run_as_root \"mkdir -p "
               + resultRoot
               + "/acceptance_test_tmp\"");
       TestUtils.exec(
           "/usr/local/packages/aime/ias/run_as_root \"chmod 777 "
               + resultRoot
               + "/acceptance_test_tmp\"");
-      logger.info("Creating " + pvRoot + "/acceptance_test_pv");
+      logger.info("Deleting and Creating " + pvRoot + "/acceptance_test_pv");
       TestUtils.exec(
-          "/usr/local/packages/aime/ias/run_as_root \"mkdir -p "
+          "/usr/local/packages/aime/ias/run_as_root \"rm -rf "
+              + pvRoot
+              + "/acceptance_test_pv\" && "
+              + "/usr/local/packages/aime/ias/run_as_root \"mkdir -p "
               + pvRoot
               + "/acceptance_test_pv\"");
       TestUtils.exec(
@@ -177,7 +187,7 @@ public static void initialize(String appPropsFile) throws Exception {
     logger.info("Adding file handler, logging to file at " + resultDir + "/java_test_suite.out");
 
     // for manual/local run, create file handler, create PVROOT
-    if (System.getenv("SHARED_CLUSTER") == null && System.getenv("JENKINS") == null) {
+    if (!JENKINS && !SHARED_CLUSTER) {
       logger.info("Creating PVROOT " + pvRoot);
       Files.createDirectories(Paths.get(pvRoot));
       ExecResult result = ExecCommand.exec("chmod 777 " + pvRoot);
@@ -211,6 +221,9 @@ public static void initialize(String appPropsFile) throws Exception {
         "Env var IMAGE_PULL_SECRET_OPERATOR " + System.getenv("IMAGE_PULL_SECRET_OPERATOR"));
     logger.info(
         "Env var IMAGE_PULL_SECRET_WEBLOGIC " + System.getenv("IMAGE_PULL_SECRET_WEBLOGIC"));
+    logger.info("Env var IMAGE_NAME_WEBLOGIC " + System.getenv("IMAGE_NAME_WEBLOGIC"));
+    logger.info("Env var IMAGE_TAG_WEBLOGIC " + System.getenv("IMAGE_TAG_WEBLOGIC"));
+
     logger.info("Env var BRANCH_NAME " + System.getenv("BRANCH_NAME"));
   }
 
@@ -531,7 +544,7 @@ public static ExecResult cleanup() throws Exception {
             + getResultRoot()
             + " export PV_ROOT="
             + getPvRoot()
-            + " && "
+            + " export SHARED_CLUSTER=false && "
             + getProjectRoot()
             + "/src/integration-tests/bash/cleanup.sh";
     logger.info("Command to call cleanup script " + cmd);
@@ -672,11 +685,8 @@ public static void tearDown(String iTClassName) throws Exception {
     StringBuffer cmd =
         new StringBuffer(
             "export RESULT_ROOT=$RESULT_ROOT && export PV_ROOT=$PV_ROOT && export IT_CLASS=");
-    cmd.append(iTClassName);
-    if (JENKINS) {
-      cmd.append(" && export JENKINS_RESULTS_DIR=${WORKSPACE}/logdir/${BUILD_TAG} ");
-    }
-    cmd.append(" && ")
+    cmd.append(iTClassName)
+        .append(" && export JENKINS_RESULTS_DIR=${WORKSPACE}/logdir/${BUILD_TAG} && ")
         .append(BaseTest.getProjectRoot())
         .append("/integration-tests/src/test/resources/statedump.sh");
     logger.info("Running " + cmd);
@@ -693,7 +703,7 @@ public static void tearDown(String iTClassName) throws Exception {
 
     TestUtils.renewK8sClusterLease(getProjectRoot(), getLeaseId());
 
-    if (JENKINS) {
+    if (JENKINS || SHARED_CLUSTER) {
       result = cleanup();
       if (result.exitValue() != 0) {
         logger.info("cleanup result =" + result.stdout() + "\n " + result.stderr());

integration-tests/src/test/java/oracle/kubernetes/operator/ITPodsRestart.java

@@ -200,11 +200,38 @@ public void testServerPodsRestartByChangingZImage() throws Exception {
               + domain.getDomainUid()
               + "  Image property: store/oracle/weblogic:12.2.1.3 to store/oracle/weblogic:duplicate");
 
-      TestUtils.exec("docker tag store/oracle/weblogic:12.2.1.3 store/oracle/weblogic:duplicate");
-      domain.verifyDomainServerPodRestart(
-          "\"store/oracle/weblogic:12.2.1.3\"", "\"store/oracle/weblogic:duplicate\"");
+      if (BaseTest.SHARED_CLUSTER) {
+        String newImage =
+            System.getenv("REPO_REGISTRY") + "/weblogick8s/store/oracle/weblogic:duplicate";
+
+        // tag image with repo name
+        TestUtils.exec("docker tag store/oracle/weblogic:12.2.1.3 " + newImage);
+
+        // login and push image to ocir
+        TestUtils.loginAndPushImageToOCIR(newImage);
+
+        // create ocir registry secret in the same ns as domain which is used while pulling the
+        // image
+        TestUtils.createDockerRegistrySecret(
+            "docker-store",
+            System.getenv("REPO_REGISTRY"),
+            System.getenv("REPO_USERNAME"),
+            System.getenv("REPO_PASSWORD"),
+            System.getenv("REPO_EMAIL"),
+            domain.getDomainNS());
+
+        // apply new domain yaml and verify pod restart
+        domain.verifyDomainServerPodRestart(
+            "\"store/oracle/weblogic:12.2.1.3\"", "\"" + newImage + "\"");
+      } else {
+        TestUtils.exec("docker tag store/oracle/weblogic:12.2.1.3 store/oracle/weblogic:duplicate");
+        domain.verifyDomainServerPodRestart(
+            "\"store/oracle/weblogic:12.2.1.3\"", "\"store/oracle/weblogic:duplicate\"");
+      }
     } finally {
-      TestUtils.exec("docker rmi -f store/oracle/weblogic:duplicate");
+      if (!BaseTest.SHARED_CLUSTER) {
+        TestUtils.exec("docker rmi -f store/oracle/weblogic:duplicate");
+      }
     }
 
     logger.info("SUCCESS - " + testMethodName);

integration-tests/src/test/java/oracle/kubernetes/operator/ITServerDiscovery.java

@@ -193,8 +193,6 @@ public void testOPReconnToDomain() throws Exception {
   }
 
   private void scaleDownAndverify(int decrNum) throws Exception {
-    String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
-    logTestBegin(testMethodName);
 
     Map<String, Object> domainMap = domain.getDomainMap();
     String domainUid = domain.getDomainUid();

integration-tests/src/test/java/oracle/kubernetes/operator/ITSessionMigration.java

@@ -63,7 +63,9 @@ public static void staticPrepare() throws Exception {
       // create domain
       if (domain == null) {
         logger.info("Creating WLS Domain & waiting for the script to complete execution");
-        domain = TestUtils.createDomain(DOMAINONPV_WLST_YAML);
+        Map<String, Object> wlstDomainMap = TestUtils.loadYaml(DOMAINONPV_WLST_YAML);
+        wlstDomainMap.put("domainUID", "sessmigdomainonpvwlst");
+        domain = TestUtils.createDomain(wlstDomainMap);
         domain.verifyDomainCreated();
       }
 

integration-tests/src/test/java/oracle/kubernetes/operator/ITUsabilityOperatorHelmChart.java

@@ -630,6 +630,10 @@ public void testAddRemoveDomainUpdateOperatorHC() throws Exception {
       logger.info("verify that old domain is not managed by operator");
       verifyOperatorDomainManagement(operator, domain, false);
       verifyOperatorDomainManagement(operator, domainnew, true);
+      logger.info("Upgrade to add first domain namespace in target domains");
+      targetDomainsNS.add("test" + (number));
+      upgradeOperatorDomainNamespaces(operator, targetDomainsNS);
+      verifyOperatorDomainManagement(operator, domain, true);
       testCompletedSuccessfully = true;
     } finally {
       if (domain != null) {
@@ -700,6 +704,7 @@ private void verifyOperatorDomainManagement(
               .contains(
                   "Response {\"status\":404,\"detail\":\"/operator/latest/domains/test" + number)) {
           } else {
+            logger.info("Got 404, Operator can not access the domain " + domain.getDomainUid());
             break;
           }
         }