Merge branch 'develop' into owls-73691

Author: alai8

docs/domains/Domain.json

@@ -407,7 +407,7 @@
       "type": "object",
       "properties": {
         "overallHealth": {
-          "description": "Server health of this WebLogic server.",
+          "description": "Server health of this WebLogic server. If the value is \"Not available\", the operator has failed to read the health. If the value is \"Not available (possibly overloaded)\", the operator has failed to read the health of the server possibly due to the server is in overloaded state",
           "type": "string"
         },
         "activationTime": {

docs/domains/Domain.md

@@ -173,7 +173,7 @@ Shutdown describes the configuration for shutting down a server instance.
 | Name | Type | Description |
 | --- | --- | --- |
 | `activationTime` | DateTime | RFC 3339 date and time at which the server started. |
-| `overallHealth` | string | Server health of this WebLogic server. |
+| `overallHealth` | string | Server health of this WebLogic server. If the value is "Not available", the operator has failed to read the health. If the value is "Not available (possibly overloaded)", the operator has failed to read the health of the server possibly due to the server is in overloaded state |
 | `subsystems` | array of [Subsystem Health](#subsystem-health) | Status of unhealthy subsystems, if any. |
 
 ### Channel

docs/domains/index.html

@@ -1327,7 +1327,7 @@
       "type": "object",
       "properties": {
         "overallHealth": {
-          "description": "Server health of this WebLogic server.",
+          "description": "Server health of this WebLogic server. If the value is \"Not available\", the operator has failed to read the health. If the value is \"Not available (possibly overloaded)\", the operator has failed to read the health of the server possibly due to the server is in overloaded state",
           "type": "string"
         },
         "activationTime": {

model/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerHealth.java

@@ -23,7 +23,11 @@ public class ServerHealth {
   @Expose
   private DateTime activationTime;
 
-  @Description("Server health of this WebLogic server.")
+  @Description(
+      "Server health of this WebLogic server. If the value is \"Not available\", the operator has "
+          + "failed to read the health. If the value is \"Not available (possibly overloaded)\", the "
+          + "operator has failed to read the health of the server possibly due to the server is "
+          + "in overloaded state")
   @SerializedName("overallHealth")
   @Expose
   private String overallHealth;

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

@@ -21,6 +21,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.annotation.Nullable;
 import oracle.kubernetes.operator.TuningParameters.MainTuning;
 import oracle.kubernetes.operator.calls.CallResponse;
@@ -314,7 +315,9 @@ public void run() {
                   new CompletionCallback() {
                     @Override
                     public void onCompletion(Packet packet) {
-                      if (Boolean.TRUE.equals(packet.get(ProcessingConstants.SERVER_HEALTH_READ))) {
+                      AtomicInteger serverHealthRead =
+                          packet.getValue(ProcessingConstants.REMAINING_SERVERS_HEALTH_TO_READ);
+                      if (serverHealthRead == null || serverHealthRead.get() == 0) {
                         loggingFilter.setFiltering(false).resetLogHistory();
                       } else {
                         loggingFilter.setFiltering(true);

operator/src/main/java/oracle/kubernetes/operator/PodWatcher.java

@@ -8,10 +8,11 @@
 import io.kubernetes.client.models.V1ObjectMeta;
 import io.kubernetes.client.models.V1Pod;
 import io.kubernetes.client.util.Watch;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.atomic.AtomicBoolean;
 import oracle.kubernetes.operator.TuningParameters.WatchTuning;
@@ -39,8 +40,24 @@ public class PodWatcher extends Watcher<V1Pod>
   private final WatchListener<V1Pod> listener;
 
   // Map of Pod name to OnReady
-  private final ConcurrentMap<String, OnReady> readyCallbackRegistrations =
-      new ConcurrentHashMap<>();
+  private final Map<String, Collection<OnReady>> readyCallbackRegistrations = new HashMap<>();
+
+  private void registerOnReady(String podName, OnReady onReady) {
+    synchronized (readyCallbackRegistrations) {
+      Collection<OnReady> col = readyCallbackRegistrations.get(podName);
+      if (col == null) {
+        col = new ArrayList<>();
+        readyCallbackRegistrations.put(podName, col);
+      }
+      col.add(onReady);
+    }
+  }
+
+  private Collection<OnReady> retrieveOnReady(String podName) {
+    synchronized (readyCallbackRegistrations) {
+      return readyCallbackRegistrations.remove(podName);
+    }
+  }
 
   /**
    * Factory for PodWatcher.
@@ -96,9 +113,11 @@ public void receivedResponse(Watch.Response<V1Pod> item) {
         Boolean isReady = !PodHelper.isDeleting(pod) && PodHelper.isReady(pod);
         String podName = pod.getMetadata().getName();
         if (isReady) {
-          OnReady ready = readyCallbackRegistrations.remove(podName);
-          if (ready != null) {
-            ready.onReady();
+          Collection<OnReady> col = retrieveOnReady(podName);
+          if (col != null) {
+            for (OnReady ready : col) {
+              ready.onReady();
+            }
           }
         }
         break;
@@ -148,7 +167,7 @@ public NextAction apply(Packet packet) {
                     fiber.resume(packet);
                   }
                 };
-            readyCallbackRegistrations.put(metadata.getName(), ready);
+            registerOnReady(metadata.getName(), ready);
 
             // Timing window -- pod may have come ready before registration for callback
             CallBuilderFactory factory =

operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java

@@ -32,5 +32,5 @@ public interface ProcessingConstants {
   public static final String DOMAIN_INTROSPECTOR_LOG_RESULT = "domainIntrospectorLogResult";
   public static final String SIT_CONFIG_MAP = "sitConfigMap";
 
-  public static final String SERVER_HEALTH_READ = "serverHealthRead";
+  public static final String REMAINING_SERVERS_HEALTH_TO_READ = "serverHealthRead";
 }

operator/src/main/java/oracle/kubernetes/operator/ServerStatusReader.java

@@ -21,6 +21,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import oracle.kubernetes.operator.helpers.ClientPool;
@@ -68,6 +69,9 @@ public NextAction apply(Packet packet) {
       packet.put(SERVER_STATE_MAP, new ConcurrentHashMap<String, String>());
       packet.put(SERVER_HEALTH_MAP, new ConcurrentHashMap<String, ServerHealth>());
 
+      AtomicInteger remainingServerHealthToRead = new AtomicInteger();
+      packet.put(ProcessingConstants.REMAINING_SERVERS_HEALTH_TO_READ, remainingServerHealthToRead);
+
       Collection<StepAndPacket> startDetails =
           info.getServerPods()
               .map(pod -> createStatusReaderStep(packet, pod))
@@ -76,6 +80,7 @@ public NextAction apply(Packet packet) {
       if (startDetails.isEmpty()) {
         return doNext(packet);
       } else {
+        remainingServerHealthToRead.set(startDetails.size());
         return doForkJoin(getNext(), packet, startDetails);
       }
     }

operator/src/main/java/oracle/kubernetes/operator/Watcher.java

@@ -143,7 +143,7 @@ private void watchForEvents() {
             new WatchBuilder()
                 .withResourceVersion(resourceVersion.toString())
                 .withTimeoutSeconds(tuning.watchLifetime))) {
-      while (watch.hasNext()) {
+      while (hasNext(watch)) {
         Watch.Response<T> item = watch.next();
 
         if (isStopping()) {
@@ -164,6 +164,15 @@ private void watchForEvents() {
     }
   }
 
+  private boolean hasNext(WatchI<T> watch) {
+    try {
+      return watch.hasNext();
+    } catch (Throwable ex) {
+      // no-op on exception during hasNext
+    }
+    return false;
+  }
+
   /**
    * Initiates a watch by using the watch builder to request any updates for the specified watcher.
    *

operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java

@@ -190,7 +190,7 @@ public void onSuccess(
                         try {
                           cc.cancel();
                         } finally {
-                          LOGGER.info(
+                          LOGGER.fine(
                               MessageKeys.ASYNC_TIMEOUT,
                               requestParams.call,
                               requestParams.namespace,