Merge pull request #854 from oracle/feature/OWLS-70864-integ

Test Network Access Point on WL Clusters

Author: rjeberhard

integration-tests/src/test/java/oracle/kubernetes/operator/BaseTest.java

@@ -251,6 +251,7 @@ public void testAdminT3ChannelWithJMS(Domain domain) throws Exception {
     c.close();
     logger.info("Done - testAdminT3ChannelWithJMS");
   }
+
   /**
    * Restarting the domain should not have any impact on Operator managing the domain, web app load
    * balancing and node port service
@@ -271,6 +272,7 @@ public void testDomainLifecyle(Operator operator, Domain domain) throws Exceptio
       domain.verifyWebAppLoadBalancing(TESTWEBAPP);
     }
     domain.verifyAdminServerExternalService(getUsername(), getPassword());
+    domain.verifyHasClusterServiceChannelPort("TCP", 8011, TESTWEBAPP + "/");
     logger.info("Done - testDomainLifecyle");
   }
 
@@ -315,22 +317,23 @@ public void testClusterScaling(Operator operator, Domain domain) throws Exceptio
 
     // make sure cluster service endpoint is updated with the scaled up servers before verifying
     // load balancing
-    int i = 0;
-    while (i < BaseTest.getMaxIterationsPod()) {
-      int numberOfServersInEndpoint =
-          domain.getNumberOfServersInClusterServiceEndpoint((String) domainMap.get("clusterName"));
-      if (replicaCnt != numberOfServersInEndpoint) {
-        // check for last iteration
-        if (i == (BaseTest.getMaxIterationsPod() - 1)) {
-          throw new RuntimeException("FAILURE: Cluster Serice Endpoint is not updated");
-        }
-        Thread.sleep(BaseTest.getWaitTimePod() * 1000);
-        i++;
-      } else {
-        logger.info("Number of servers addresses in endpoint is same as replica count ");
-        break;
-      }
-    }
+    //    int i = 0;
+    //    while (i < BaseTest.getMaxIterationsPod()) {
+    //      int numberOfServersInEndpoint =
+    //          domain.getNumberOfServersInClusterServiceEndpoint((String)
+    // domainMap.get("clusterName"));
+    //      if (replicaCnt != numberOfServersInEndpoint) {
+    //        // check for last iteration
+    //        if (i == (BaseTest.getMaxIterationsPod() - 1)) {
+    //          throw new RuntimeException("FAILURE: Cluster Serice Endpoint is not updated");
+    //        }
+    //        Thread.sleep(BaseTest.getWaitTimePod() * 1000);
+    //        i++;
+    //      } else {
+    //        logger.info("Number of servers addresses in endpoint is same as replica count ");
+    //        break;
+    //      }
+    //    }
     // commenting the load balance check, bug 29325139
     // domain.verifyWebAppLoadBalancing(TESTWEBAPP);
 

integration-tests/src/test/java/oracle/kubernetes/operator/ITOperator.java

@@ -433,6 +433,9 @@ public void testAutoAndCustomSitConfigOverrides() throws Exception {
       // load input yaml to map and add configOverrides
       Map<String, Object> domainMap = TestUtils.loadYaml(domainonpvwlstFile);
       domainMap.put("configOverrides", "sitconfigcm");
+      domainMap.put(
+          "configOverridesFile",
+          "/integration-tests/src/test/resources/domain-home-on-pv/customsitconfig");
       domainMap.put("domainUID", "customsitdomain");
       domainMap.put("adminNodePort", new Integer("30704"));
       domainMap.put("t3ChannelPort", new Integer("30051"));

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Domain.java

@@ -234,6 +234,7 @@ public void verifyServersReady() throws Exception {
       }
     }
   }
+
   /**
    * verify nodeport by accessing admin REST endpoint
    *
@@ -280,6 +281,73 @@ public void verifyAdminServerExternalService(String username, String password) t
     }
   }
 
+  /**
+   * Verify that we have the channel set for the cluster
+   *
+   * @param protocol
+   * @param port
+   * @param path
+   * @throws Exception
+   */
+  public void verifyHasClusterServiceChannelPort(String protocol, int port, String path)
+      throws Exception {
+
+    /* Make sure the service exists in k8s */
+    if (!TestUtils.checkHasServiceChannelPort(
+        this.getDomainUid() + "-cluster-" + this.clusterName, domainNS, protocol, port)) {
+      throw new RuntimeException(
+          "FAILURE: Cannot find channel port in cluster, but expecting one: "
+              + port
+              + "/"
+              + protocol);
+    }
+
+    /*
+     * Construct a curl command that will be run via kubectl exec on the admin server
+     */
+    StringBuffer curlCmd =
+        new StringBuffer(
+            "kubectl exec " + this.getDomainUid() + "-" + this.adminServerName + " /usr/bin/curl ");
+
+    /*
+     * Make sure we can reach the port,
+     * first via each managed server URL
+     */
+    for (int i = 1; i <= TestUtils.getClusterReplicas(domainUid, clusterName, domainNS); i++) {
+      StringBuffer serverAppUrl = new StringBuffer("http://");
+      serverAppUrl
+          .append(this.getDomainUid() + "-" + managedServerNameBase + i)
+          .append(":")
+          .append(port)
+          .append("/");
+      serverAppUrl.append(path);
+
+      callWebAppAndWaitTillReady(
+          new StringBuffer(curlCmd.toString())
+              .append(serverAppUrl.toString())
+              .append(" -- --write-out %{http_code} -o /dev/null")
+              .toString());
+    }
+
+    /*
+     * Make sure we can reach the port,
+     * second via cluster URL which should round robin through each managed server.
+     * Use the callWebAppAndCheckForServerNameInResponse method with verifyLoadBalancing
+     * enabled to verify each managed server is responding.
+     */
+    StringBuffer clusterAppUrl = new StringBuffer("http://");
+    clusterAppUrl
+        .append(this.getDomainUid() + "-cluster-" + this.clusterName)
+        .append(":")
+        .append(port)
+        .append("/");
+    clusterAppUrl.append(path);
+
+    // execute curl and look for each managed server name in response
+    callWebAppAndCheckForServerNameInResponse(
+        new StringBuffer(curlCmd.toString()).append(clusterAppUrl.toString()).toString(), true);
+  }
+
   /**
    * deploy webapp using nodehost and nodeport
    *
@@ -416,8 +484,8 @@ public void callWebAppAndVerifyLoadBalancing(String webappName, boolean verifyLo
       StringBuffer curlCmdResCode = new StringBuffer(curlCmd.toString());
       curlCmdResCode.append(" --write-out %{http_code} -o /dev/null");
 
-      logger.info("Curd cmd with response code " + curlCmdResCode);
-      logger.info("Curd cmd " + curlCmd);
+      logger.info("Curl cmd with response code " + curlCmdResCode);
+      logger.info("Curl cmd " + curlCmd);
 
       // call webapp iteratively till its deployed/ready
       callWebAppAndWaitTillReady(curlCmdResCode.toString());
@@ -1102,9 +1170,12 @@ private void initialize(Map<String, Object> inputDomainMap) throws Exception {
     domainMap.values().removeIf(Objects::isNull);
 
     // create config map and secret for custom sit config
-    if (domainMap.get("configOverrides") != null) {
+    if ((domainMap.get("configOverrides") != null)
+        && (domainMap.get("configOverridesFile") != null)) {
       // write hostname in config file for public address
 
+      String configOverridesFile = domainMap.get("configOverridesFile").toString();
+
       String cmd =
           "kubectl -n "
               + domainNS
@@ -1114,7 +1185,7 @@ private void initialize(Map<String, Object> inputDomainMap) throws Exception {
               + domainMap.get("configOverrides")
               + " --from-file "
               + BaseTest.getProjectRoot()
-              + "/integration-tests/src/test/resources/domain-home-on-pv/customsitconfig";
+              + configOverridesFile;
       ExecResult result = ExecCommand.exec(cmd);
       if (result.exitValue() != 0) {
         throw new RuntimeException(

integration-tests/src/test/java/oracle/kubernetes/operator/utils/TestUtils.java

@@ -220,6 +220,39 @@ public static boolean checkPVReleased(String pvBaseName, String namespace) throw
     }
     return true;
   }
+
+  /**
+   * NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) domain1-cluster-cluster-1 ClusterIP 10.105.146.61
+   * <none> 30032/TCP,8001/TCP domain1-managed-server1 ClusterIP None <none> 30032/TCP,8001/TCP
+   *
+   * @param service
+   * @param namespace
+   * @param protocol
+   * @param port
+   * @return
+   * @throws Exception
+   */
+  public static boolean checkHasServiceChannelPort(
+      String service, String namespace, String protocol, int port) throws Exception {
+    StringBuffer cmd = new StringBuffer("kubectl get services ");
+    cmd.append(" -n ").append(namespace);
+    logger.info(" Find services in namespage " + namespace + " with command: '" + cmd + "'");
+
+    ExecResult result = ExecCommand.exec(cmd.toString());
+    String stdout = result.stdout();
+    logger.info(" Services found: ");
+    logger.info(stdout);
+    String stdoutlines[] = stdout.split("\\r?\\n");
+    if (result.exitValue() == 0 && stdoutlines.length > 0) {
+      for (String stdoutline : stdoutlines) {
+        if (stdoutline.contains(service) && stdoutline.contains(port + "/" + protocol)) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
   /**
    * First, kill the mgd server process in the container three times to cause the node manager to
    * mark the server 'failed not restartable'. This in turn is detected by the liveness probe, which

integration-tests/src/test/resources/domain-home-on-pv/create-domain.py

@@ -100,8 +100,15 @@ def getEnvVar(var):
   cd('/ServerTemplates/%s' % templateName)
   cmo.setListenPort(server_port)
   cmo.setCluster(cl)
-  print('Done setting attributes for Server Template: %s' % templateName);
 
+  templateChannelName = cluster_name + "-NAP"
+  print('Creating Server Template NAP: %s' % cluster_name + "-NAP")
+  create(templateChannelName, 'NetworkAccessPoint')
+  cd('NetworkAccessPoints/%s' % templateChannelName)
+  set('PublicPort', server_port + 10)
+  set('ListenPort', server_port + 10)
+  print('Done creating Server Template NAP: %s' % cluster_name + "-NAP")
+  print('Done setting attributes for Server Template: %s' % templateName);
 
   cd('/Clusters/%s' % cluster_name)
   create(cluster_name, 'DynamicServers')