Merge branch 'elk-debug-r34' into 'release/3.4'

OWLS-110915 - Backport the fix for the failure of testOpLogLevelSearch in ItElasticLogging to release/3.4

See merge request weblogic-cloud/weblogic-kubernetes-operator!4354

Author: rjeberhard

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItElasticLogging.java

@@ -26,7 +26,6 @@
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable;
 
 import static oracle.weblogic.kubernetes.TestConstants.COPY_WLS_LOGGING_EXPORTER_FILE_NAME;
 import static oracle.weblogic.kubernetes.TestConstants.ELASTICSEARCH_HTTPS_PORT;
@@ -57,7 +56,7 @@
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.operatorIsReady;
 import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createMiiDomainAndVerify;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
-import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withStandardRetryPolicy;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withLongRetryPolicy;
 import static oracle.weblogic.kubernetes.utils.ExecCommand.exec;
 import static oracle.weblogic.kubernetes.utils.FileUtils.copyFileFromPodUsingK8sExec;
 import static oracle.weblogic.kubernetes.utils.FileUtils.searchStringInFile;
@@ -282,7 +281,7 @@ void testLogLevelSearch() {
     String queryCriteria = "/_count?q=level:INFO";
 
     // verify log level query results
-    withStandardRetryPolicy.untilAsserted(
+    withLongRetryPolicy.untilAsserted(
         () -> assertTrue(verifyCountsHitsInSearchResults(queryCriteria, regex, LOGSTASH_INDEX_KEY, true),
             String.format("Query logs of level=INFO failed")));
 
@@ -300,9 +299,12 @@ void testOperatorLogSearch() {
     String regex = ".*took\":(\\d+),.*hits\":\\{(.+)\\}";
     String queryCriteria = "/_search?q=type:weblogic-operator";
 
-    verifyCountsHitsInSearchResults(queryCriteria, regex, LOGSTASH_INDEX_KEY, false);
+    // verify results of query of type:weblogic-operator in Operator log
+    withLongRetryPolicy.untilAsserted(
+        () -> assertTrue(verifyCountsHitsInSearchResults(queryCriteria, regex, LOGSTASH_INDEX_KEY, false),
+            "Query Operator log info q=type:weblogic-operator failed"));
 
-    logger.info("Query Operator log info succeeded");
+    logger.info("Query Operator log info q=type:weblogic-operator succeeded");
   }
 
   /**
@@ -326,9 +328,9 @@ void testWebLogicLogSearch() {
    * Use Elasticsearch Search APIs to query WebLogic log info pushed to Elasticsearch repository
    * by WebLogic Logging Exporter. Verify that log occurrence for WebLogic servers are not empty.
    */
+  @Disabled("WlsLoggingExporter is not supported any more")
   @Test
   @DisplayName("Use Elasticsearch Search APIs to query WebLogic log info in WLS server pod and verify")
-  @DisabledIfEnvironmentVariable(named = "OKD", matches = "true")
   void testWlsLoggingExporter() throws Exception {
     Map<String, String> wlsMap = verifyLoggingExporterReady(opNamespace, elasticSearchNs, null, WEBLOGIC_INDEX_KEY);
     // merge testVarMap and wlsMap
@@ -422,7 +424,7 @@ private static  void createAndVerifyDomain(String miiImage) {
 
   private void verifyServerRunningInSearchResults(String serverName) {
     String queryCriteria = "/_search?q=log:" + serverName;
-    withStandardRetryPolicy.untilAsserted(
+    withLongRetryPolicy.untilAsserted(
         () -> assertTrue(execSearchQuery(queryCriteria, LOGSTASH_INDEX_KEY).contains("RUNNING"),
           String.format("serverName %s is not RUNNING", serverName)));
 
@@ -531,7 +533,7 @@ private void verifyLogstashConfigMapModifyResult(String replaceStr) {
     }
 
     // wait for logstash config modified and verify
-    withStandardRetryPolicy.untilAsserted(
+    withLongRetryPolicy.untilAsserted(
         () -> assertTrue(copyConfigFromPodAndSearchForString(containerName, replaceStr),
             String.format("Failed to find search string %s", replaceStr)));
   }

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItElasticLoggingFluentd.java

@@ -39,7 +39,6 @@
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.DisplayName;
-import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT;
@@ -101,7 +100,6 @@
  */
 @DisplayName("Test to use Elasticsearch API to query WebLogic logs")
 @IntegrationTest
-@Tag("kind-parallel")
 class ItElasticLoggingFluentd {
 
   // constants for creating domain image using model in image
@@ -171,8 +169,8 @@ public static void init(@Namespaces(3) List<String> namespaces) {
     // install and verify Operator
     installAndVerifyOperator(opNamespace, opNamespace + "-sa",
         false, 0, true, domainNamespace);
-
-    elasticSearchHost = "elasticsearch." + elasticSearchNs + ".svc.cluster.local";
+    
+    elasticSearchHost = "elasticsearch." + elasticSearchNs + ".svc";
 
     // upgrade to latest operator
     HelmParams upgradeHelmParams = new HelmParams()

integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/LoggingExporter.java

@@ -518,7 +518,7 @@ private static V1Service createKibanaServiceCr(LoggingExporterParams params) {
 
   private static String execLoggingExpStatusCheck(String opNamespace, String esNamespace,
                                                   String labelSelector, String indexRegex) {
-    String elasticSearchHost = "elasticsearch." + esNamespace + ".svc.cluster.local";
+    String elasticSearchHost = "elasticsearch." + esNamespace + ".svc";
     StringBuffer k8sExecCmdPrefixBuff = new StringBuffer("curl http://");
     String cmd = k8sExecCmdPrefixBuff
         .append(elasticSearchHost)