Fix assertion and add events log to the logs list (#1619)

* Add namespaced events in the collected logs list
Fix the persistent volume claim log file name
Fix the isPodReady test assertion to check for pod readyness instead of
Running status
Change the wait time from 5 minutes to 10 minutes

* Fix the operator pod ready assertion

* Adding roles, rolebindings, clusterroles and clusterrolebindings to the
diagnostics

* Increasing the introspector pod timeout to 5 minutes

* change the namespaced roles and rolebindings file names

Author: sankarpn

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDomain.java

@@ -81,7 +81,7 @@
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.doesImageExist;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainExists;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.isHelmReleaseDeployed;
-import static oracle.weblogic.kubernetes.assertions.TestAssertions.operatorIsRunning;
+import static oracle.weblogic.kubernetes.assertions.TestAssertions.operatorIsReady;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.podExists;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.podReady;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.serviceExists;
@@ -225,7 +225,7 @@ public static void initAll(@Namespaces(3) List<String> namespaces) {
                 opNamespace,
                 condition.getElapsedTimeInMS(),
                 condition.getRemainingTimeInMS()))
-        .until(operatorIsRunning(opNamespace));
+        .until(operatorIsReady(opNamespace));
 
   }
 
@@ -666,7 +666,8 @@ private void createDomainResource(String domainUid, String domNamespace, String
                     .configuration(new Configuration()
                             .model(new Model()
                                     .domainType("WLS")
-                                    .runtimeEncryptionSecret(encryptionSecretName))));
+                                    .runtimeEncryptionSecret(encryptionSecretName))
+                        .introspectorJobActiveDeadlineSeconds(300L)));
 
     logger.info("Create domain custom resource for domainUid {0} in namespace {1}",
             domainUid, domNamespace);

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItSimpleOperatorValidation.java

@@ -40,7 +40,7 @@
 import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorImageName;
 import static oracle.weblogic.kubernetes.actions.TestActions.installOperator;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.isHelmReleaseDeployed;
-import static oracle.weblogic.kubernetes.assertions.TestAssertions.operatorIsRunning;
+import static oracle.weblogic.kubernetes.assertions.TestAssertions.operatorIsReady;
 import static org.awaitility.Awaitility.with;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -154,7 +154,7 @@ public void testInstallingOperator(@Namespaces(3) List<String> namespaces) {
                 opNamespace,
                 condition.getElapsedTimeInMS(),
                 condition.getRemainingTimeInMS()))
-        .until(operatorIsRunning(opNamespace));
+        .until(operatorIsReady(opNamespace));
 
   }
 

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/primitive/Kubernetes.java

@@ -12,6 +12,8 @@
 import java.io.OutputStream;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.concurrent.Callable;
 
@@ -42,6 +44,8 @@
 import io.kubernetes.client.openapi.models.V1ConfigMapList;
 import io.kubernetes.client.openapi.models.V1Deployment;
 import io.kubernetes.client.openapi.models.V1DeploymentList;
+import io.kubernetes.client.openapi.models.V1Event;
+import io.kubernetes.client.openapi.models.V1EventList;
 import io.kubernetes.client.openapi.models.V1Job;
 import io.kubernetes.client.openapi.models.V1JobList;
 import io.kubernetes.client.openapi.models.V1Namespace;
@@ -245,7 +249,7 @@ private static void initializeGenericKubernetesApiClients() {
             V1SecretList.class, // the api list type class
             "", // the api group
             "v1", // the api version
-            "serviceaccounts", // the resource plural
+            "secrets", // the resource plural
             apiClient //the api client
         );
 
@@ -639,6 +643,40 @@ public Boolean call() throws Exception {
     };
   }
 
+  // --------------------------- Events ---------------------------------------------------
+
+  /**
+   * List events in a namespace.
+   *
+   * @param namespace name of the namespace in which to list events
+   * @return List of {@link V1Event} objects
+   * @throws ApiException when listing events fails
+   */
+  public static List<V1Event> listNamespacedEvents(String namespace) throws ApiException {
+    List<V1Event> events = null;
+    try {
+      V1EventList list = coreV1Api.listNamespacedEvent(
+          namespace, // String | namespace.
+          PRETTY, // String | If 'true', then the output is pretty printed.
+          ALLOW_WATCH_BOOKMARKS, // Boolean | allowWatchBookmarks requests watch events with type "BOOKMARK".
+          null, // String | The continue option should be set when retrieving more results from the server.
+          null, // String | A selector to restrict the list of returned objects by their fields.
+          null, // String | A selector to restrict the list of returned objects by their labels.
+          null, // Integer | limit is a maximum number of responses to return for a list call.
+          RESOURCE_VERSION, // String | Shows changes that occur after that particular version of a resource.
+          TIMEOUT_SECONDS, // Integer | Timeout for the list call.
+          Boolean.FALSE // Boolean | Watch for changes to the described resources.
+      );
+      events = list.getItems();
+      events.sort(Comparator.comparing(e -> e.getMetadata().getCreationTimestamp()));
+      Collections.reverse(events);
+    } catch (ApiException apex) {
+      logger.warning(apex.getResponseBody());
+      throw apex;
+    }
+    return events;
+  }
+
   // --------------------------- Custom Resource Domain -----------------------------------
   /**
    * Create a Domain Custom Resource.

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java

@@ -26,8 +26,8 @@ public class TestAssertions {
    * @param namespace in which is operator is running
    * @return true if running false otherwise
    */
-  public static Callable<Boolean> operatorIsRunning(String namespace) {
-    return Operator.isRunning(namespace);
+  public static Callable<Boolean> operatorIsReady(String namespace) {
+    return Operator.isReady(namespace);
   }
 
   /**
@@ -91,10 +91,11 @@ public static Callable<Boolean> podDoesNotExist(String podName, String domainUid
    * @param domainUid WebLogic domain uid in which the pod belongs
    * @param namespace in which the pod is running
    * @return true if the pod is running otherwise false
+   * @throws ApiException when Kubernetes cluster query fails to get pod
    */
   public static Callable<Boolean> podReady(String podName, String domainUid, String namespace) throws ApiException {
     return () -> {
-      return Kubernetes.isPodRunning(namespace, domainUid, podName);
+      return Kubernetes.isPodReady(namespace, domainUid, podName);
     };
   }
 

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Kubernetes.java

@@ -15,6 +15,7 @@
 import io.kubernetes.client.openapi.apis.CoreV1Api;
 import io.kubernetes.client.openapi.apis.CustomObjectsApi;
 import io.kubernetes.client.openapi.models.V1Pod;
+import io.kubernetes.client.openapi.models.V1PodCondition;
 import io.kubernetes.client.openapi.models.V1PodList;
 import io.kubernetes.client.openapi.models.V1Service;
 import io.kubernetes.client.openapi.models.V1ServiceList;
@@ -87,6 +88,40 @@ public static boolean isPodRunning(String namespace, String domainUid, String po
     return status;
   }
 
+  /**
+   * Checks if a pod is ready in a given namespace.
+   *
+   * @param namespace in which to check if the pod is ready
+   * @param domainUid the label the pod is decorated with
+   * @param podName name of the pod to check for
+   * @return true if the pod is in the ready condition, false otherwise
+   * @throws ApiException if Kubernetes client API call fails
+   */
+  public static boolean isPodReady(String namespace, String domainUid, String podName) throws ApiException {
+    boolean status = false;
+    String labelSelector = null;
+    if (domainUid != null) {
+      labelSelector = String.format("weblogic.domainUID in (%s)", domainUid);
+    }
+
+    V1Pod pod = getPod(namespace, labelSelector, podName);
+    if (pod != null) {
+
+      // get the podCondition with the 'Ready' type field
+      V1PodCondition v1PodReadyCondition = pod.getStatus().getConditions().stream()
+          .filter(v1PodCondition -> "Ready".equals(v1PodCondition.getType()))
+          .findAny()
+          .orElse(null);
+
+      if (v1PodReadyCondition != null) {
+        status = v1PodReadyCondition.getStatus().equalsIgnoreCase("true");
+      }
+    } else {
+      logger.info("Pod doesn't exist");
+    }
+    return status;
+  }
+
   /**
    * Checks if a pod exists in a given namespace and in Terminating state.
    * @param namespace in which to check for the pod
@@ -119,12 +154,20 @@ public static boolean isPodTerminating(String namespace, String domainUid, Strin
    * @return true if pod exists and running otherwise false
    * @throws ApiException when there is error in querying the cluster
    */
-  public static boolean isOperatorPodRunning(String namespace) throws ApiException {
+  public static boolean isOperatorPodReady(String namespace) throws ApiException {
     boolean status = false;
     String labelSelector = String.format("weblogic.operatorName in (%s)", namespace);
     V1Pod pod = getPod(namespace, labelSelector, "weblogic-operator-");
     if (pod != null) {
-      status = pod.getStatus().getPhase().equals(RUNNING);
+      // get the podCondition with the 'Ready' type field
+      V1PodCondition v1PodReadyCondition = pod.getStatus().getConditions().stream()
+          .filter(v1PodCondition -> "Ready".equals(v1PodCondition.getType()))
+          .findAny()
+          .orElse(null);
+
+      if (v1PodReadyCondition != null) {
+        status = v1PodReadyCondition.getStatus().equalsIgnoreCase("true");
+      }
     } else {
       logger.info("Pod doesn't exist");
     }
@@ -155,10 +198,10 @@ public static V1Pod getPod(String namespace, String labelSelector, String podNam
         );
     for (V1Pod item : v1PodList.getItems()) {
       if (item.getMetadata().getName().startsWith(podName.trim())) {
-        logger.info("Pod Name :" + item.getMetadata().getName());
-        logger.info("Pod Namespace :" + item.getMetadata().getNamespace());
-        logger.info("Pod UID :" + item.getMetadata().getUid());
-        logger.info("Pod Status :" + item.getStatus().getPhase());
+        logger.info("Pod Name: " + item.getMetadata().getName());
+        logger.info("Pod Namespace: " + item.getMetadata().getNamespace());
+        logger.info("Pod UID: " + item.getMetadata().getUid());
+        logger.info("Pod Status: " + item.getStatus().getPhase());
         return item;
       }
     }

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Operator.java

@@ -16,9 +16,9 @@ public class Operator {
    * @param namespace in which to check for the operator pod
    * @return true if found and running otherwise false
    */
-  public static Callable<Boolean> isRunning(String namespace) {
+  public static Callable<Boolean> isReady(String namespace) {
     return () -> {
-      return Kubernetes.isOperatorPodRunning(namespace);
+      return Kubernetes.isOperatorPodReady(namespace);
     };
   }
 

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoggingUtil.java

@@ -78,6 +78,13 @@ public static void generateLog(Object itInstance, List namespaces) {
   public static void collectLogs(String namespace, String resultDir) {
     logger.info("Collecting logs in namespace : {0}", namespace);
 
+    // get events
+    try {
+      writeToFile(Kubernetes.listNamespacedEvents(namespace), resultDir, namespace + ".list.events.log");
+    } catch (Exception ex) {
+      logger.warning("Listing events failed, not collecting any data for events");
+    }
+
     // get service accounts
     try {
       writeToFile(Kubernetes.listServiceAccounts(namespace), resultDir,
@@ -130,7 +137,7 @@ public static void collectLogs(String namespace, String resultDir) {
     }
     // write pv list
     try {
-      writeToFile(pvList, resultDir, ".list.persistent-volumes.log");
+      writeToFile(pvList, resultDir, "list.persistent-volumes.log");
     } catch (IOException ex) {
       logger.warning(ex.getMessage());
     }
@@ -173,6 +180,38 @@ public static void collectLogs(String namespace, String resultDir) {
       logger.warning(ex.getMessage());
     }
 
+    // get cluster roles
+    try {
+      writeToFile(Kubernetes.listClusterRoles(null), resultDir,
+          "list.cluster-roles.log");
+    } catch (Exception ex) {
+      logger.warning(ex.getMessage());
+    }
+
+    // get cluster role bindings
+    try {
+      writeToFile(Kubernetes.listClusterRoleBindings(null), resultDir,
+          "list.cluster-rolebindings.log");
+    } catch (Exception ex) {
+      logger.warning(ex.getMessage());
+    }
+
+    // get namespaced roles
+    try {
+      writeToFile(Kubernetes.listNamespacedRoles(namespace), resultDir,
+          namespace + ".list.roles.log");
+    } catch (Exception ex) {
+      logger.warning(ex.getMessage());
+    }
+
+    // get namespaced rolebindings
+    try {
+      writeToFile(Kubernetes.listNamespacedRoleBinding(namespace), resultDir,
+          namespace + ".list.rolebindings.log");
+    } catch (Exception ex) {
+      logger.warning(ex.getMessage());
+    }
+
     // get domain objects in the given namespace
     try {
       writeToFile(Kubernetes.listDomains(namespace), resultDir, namespace + ".list.domains.log");