Merge branch 'monexpfix' into 'release/4.2'

Fixed firealert issue in MonitoringExporterSamples test - OWLS-127302

See merge request weblogic-cloud/weblogic-kubernetes-operator!4943

Author: rjeberhard

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterSamples.java

@@ -107,6 +107,7 @@
 import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.createIngressForDomainAndVerify;
 import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyNginx;
 import static oracle.weblogic.kubernetes.utils.MonitoringUtils.checkMetricsViaPrometheus;
+import static oracle.weblogic.kubernetes.utils.MonitoringUtils.checkPrometheusAlert;
 import static oracle.weblogic.kubernetes.utils.MonitoringUtils.cleanupPromGrafanaClusterRoles;
 import static oracle.weblogic.kubernetes.utils.MonitoringUtils.createAndVerifyDomain;
 import static oracle.weblogic.kubernetes.utils.MonitoringUtils.createAndVerifyMiiImage;
@@ -379,7 +380,7 @@ void testEndToEndViaChart() throws Exception {
     }
   }
 
-  private void fireAlert() throws ApiException {
+  private void fireAlert() throws Exception {
     // scale domain2
     logger.info("Scaling cluster {0} of domain {1} in namespace {2} to {3} servers.",
         cluster1Name, domain2Uid, domain2Namespace, 1);
@@ -388,6 +389,15 @@ private void fireAlert() throws ApiException {
         domain2Uid + "-" + MANAGED_SERVER_NAME_BASE, replicaCount, managedServersCount,
         null, null);
 
+    logger.info("Wait for the prometheus  to create alert ");
+    checkPrometheusAlert("ClusterWarning", hostPortPrometheus,
+        prometheusReleaseName
+            + "." + monitoringNS);
+
+    logger.info("Wait for the prometheus  to fire alert ");
+    checkPrometheusAlert("firing", hostPortPrometheus,
+        prometheusReleaseName
+            + "." + monitoringNS);
     //check webhook log for firing alert
     List<V1Pod> pods = listPods(webhookNS, "app=webhook").getItems();
     assertNotNull((pods), "No pods are running in namespace : " + webhookNS);
@@ -402,7 +412,7 @@ private void fireAlert() throws ApiException {
 
     testUntil(withLongRetryPolicy,
         assertDoesNotThrow(() -> searchPodLogForKey(pod,
-            "Some WLS cluster has only one running server for more than 1 minutes"),
+            "Some WLS cluster has only one running server for more than 15 secs"),
             "webhook failed to fire alert"),
         logger,
         "webhook to fire alert");
@@ -554,7 +564,7 @@ public void tearDownAll() {
     deletePersistentVolumeClaim("pvc-" + grafanaReleaseName, monitoringNS);
     deletePersistentVolume("pv-test" + grafanaReleaseName);
     deleteNamespace(monitoringNS);
-    uninstallDeploymentService(webhookDepl, webhookService);
+    //uninstallDeploymentService(webhookDepl, webhookService);
     uninstallDeploymentService(coordinatorDepl, coordinatorService);
     if (nginxHelmParams != null) {
       assertThat(uninstallNginx(nginxHelmParams.getHelmParams()))

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/MonitoringUtils.java

@@ -324,6 +324,33 @@ public static void checkMetricsViaPrometheus(String searchKey, String expectedVa
         expectedVal);
   }
 
+  /**
+   * Check metrics using Prometheus.
+   *
+   * @param expectedVal        - expected alert data to search
+   * @param hostPortPrometheus host:nodePort for prometheus
+   * @throws Exception if command to check metrics fails
+   */
+  public static void checkPrometheusAlert(String expectedVal,
+                                               String hostPortPrometheus, String ingressHost)
+      throws Exception {
+
+    LoggingFacade logger = getLogger();
+    // url
+    String curlCmd =
+        String.format("curl -g --silent --show-error --noproxy '*' -v  -H 'host: " + ingressHost + "'"
+                + " http://%s/api/v1/alerts",
+            hostPortPrometheus);
+
+    logger.info("Executing Curl cmd {0}", curlCmd);
+    logger.info(" expected Value {0} ", expectedVal);
+    testUntil(
+        searchForKey(curlCmd, expectedVal),
+        logger,
+        "Check prometheus alert against expected {0}",
+         expectedVal);
+  }
+
   /**
    * Check output of the command against expected output.
    *

integration-tests/src/test/resources/exporter/promvalues.yaml

@@ -143,11 +143,11 @@ serverFiles:
         rules:
           - alert: ClusterWarning
             for: 15s
-            expr: sum by(weblogic_domainUID, weblogic_clusterName) (up{weblogic_domainUID=~'.+'}) == 1
+            expr: sum by(weblogic_domainUID, weblogic_clusterName) (up{weblogic_domainUID=~'.+', job="wls-domain1"}) == 1
             labels:
               severity: page
             annotations:
-              description: 'Some WLS cluster has only one running server for more than 1 minutes.'
+              description: 'Some WLS cluster has only one running server for more than 15 secs.'
               summary: 'Some wls cluster is in warning state.'
 
 extraScrapeConfigs: |

integration-tests/src/test/resources/exporter/promvaluesoke.yaml

@@ -147,11 +147,11 @@ serverFiles:
         rules:
           - alert: ClusterWarning
             for: 15s
-            expr: sum by(weblogic_domainUID, weblogic_clusterName) (up{weblogic_domainUID=~'.+'}) == 1
+            expr: sum by(weblogic_domainUID, weblogic_clusterName) (up{weblogic_domainUID=~'.+', job="wls-domain1"}) == 1
             labels:
               severity: page
             annotations:
-              description: 'Some WLS cluster has only one running server for more than 1 minutes.'
+              description: 'Some WLS cluster has only one running server for more than 15 secs.'
               summary: 'Some wls cluster is in warning state.'
 
 extraScrapeConfigs: |