OWLS-89106 - Potential fix for pod startup issue in GBU CNE environment after node drain/repave operation (#2398)

* Fix for pod startup issue after node drain/repave operation.

Author: ankedia

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

@@ -746,7 +746,7 @@ private void addServerToMaps(Map<String, ServerHealth> serverHealthMap,
    */
   class MakeRightDomainOperationImpl implements MakeRightDomainOperation {
 
-    private final DomainPresenceInfo liveInfo;
+    private DomainPresenceInfo liveInfo;
     private boolean explicitRecheck;
     private boolean deleting;
     private boolean willInterrupt;
@@ -851,6 +851,22 @@ public void setInspectionRun() {
       inspectionRun = true;
     }
 
+    @Override
+    public void setLiveInfo(DomainPresenceInfo info) {
+      this.liveInfo = info;
+    }
+
+    @Override
+    public void clear() {
+      this.liveInfo = null;
+      this.eventData = null;
+      this.explicitRecheck = false;
+      this.deleting = false;
+      this.willInterrupt = false;
+      this.inspectionRun = false;
+    }
+
+
     @Override
     public boolean wasInspectionRun() {
       return inspectionRun;

operator/src/main/java/oracle/kubernetes/operator/JobWatcher.java

@@ -12,6 +12,7 @@
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 import javax.annotation.Nonnull;
@@ -25,13 +26,16 @@
 import io.kubernetes.client.util.Watchable;
 import oracle.kubernetes.operator.TuningParameters.WatchTuning;
 import oracle.kubernetes.operator.builders.WatchBuilder;
+import oracle.kubernetes.operator.calls.CallResponse;
 import oracle.kubernetes.operator.helpers.CallBuilder;
 import oracle.kubernetes.operator.helpers.KubernetesUtils;
 import oracle.kubernetes.operator.helpers.ResponseStep;
 import oracle.kubernetes.operator.logging.LoggingFacade;
 import oracle.kubernetes.operator.logging.LoggingFactory;
 import oracle.kubernetes.operator.logging.MessageKeys;
+import oracle.kubernetes.operator.steps.DefaultResponseStep;
 import oracle.kubernetes.operator.watcher.WatchListener;
+import oracle.kubernetes.operator.work.NextAction;
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
 import oracle.kubernetes.utils.SystemClock;
@@ -243,6 +247,11 @@ boolean isReady(V1Job job) {
       return isComplete(job) || isFailed(job);
     }
 
+    @Override
+    boolean onReadNotFoundForCachedResource(V1Job cachedJob, boolean isNotFoundOnRead) {
+      return false;
+    }
+
     // Ignore modified callbacks from different jobs (identified by having different creation times) or those
     // where the job is not yet ready.
     @Override
@@ -302,6 +311,21 @@ Throwable createTerminationException(V1Job job) {
     void logWaiting(String name) {
       LOGGER.fine(MessageKeys.WAITING_FOR_JOB_READY, name);
     }
+
+    @Override
+    protected DefaultResponseStep<V1Job> resumeIfReady(Callback callback) {
+      return new DefaultResponseStep<>(null) {
+        @Override
+        public NextAction onSuccess(Packet packet, CallResponse<V1Job> callResponse) {
+          if (isReady(callResponse.getResult()) || callback.didResumeFiber()) {
+            callback.proceedFromWait(callResponse.getResult());
+            return doNext(packet);
+          }
+          return doDelay(createReadAndIfReadyCheckStep(callback), packet,
+                  getWatchBackstopRecheckDelaySeconds(), TimeUnit.SECONDS);
+        }
+      };
+    }
   }
 
   static class DeadlineExceededException extends Exception {

operator/src/main/java/oracle/kubernetes/operator/MakeRightDomainOperation.java

@@ -36,6 +36,10 @@ public interface MakeRightDomainOperation {
 
   void setInspectionRun();
 
+  void setLiveInfo(DomainPresenceInfo info);
+
+  void clear();
+
   boolean wasInspectionRun();
 
   private static boolean wasInspectionRun(Packet packet) {

operator/src/main/java/oracle/kubernetes/operator/PodWatcher.java

@@ -11,6 +11,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 import javax.annotation.Nonnull;
@@ -28,17 +29,26 @@
 import io.kubernetes.client.util.Watchable;
 import oracle.kubernetes.operator.TuningParameters.WatchTuning;
 import oracle.kubernetes.operator.builders.WatchBuilder;
+import oracle.kubernetes.operator.calls.CallResponse;
 import oracle.kubernetes.operator.helpers.CallBuilder;
+import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
 import oracle.kubernetes.operator.helpers.KubernetesUtils;
 import oracle.kubernetes.operator.helpers.LegalNames;
 import oracle.kubernetes.operator.helpers.PodHelper;
 import oracle.kubernetes.operator.helpers.ResponseStep;
 import oracle.kubernetes.operator.logging.LoggingFacade;
 import oracle.kubernetes.operator.logging.LoggingFactory;
 import oracle.kubernetes.operator.logging.MessageKeys;
+import oracle.kubernetes.operator.steps.DefaultResponseStep;
 import oracle.kubernetes.operator.watcher.WatchListener;
+import oracle.kubernetes.operator.work.NextAction;
+import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
 
+import static oracle.kubernetes.operator.ProcessingConstants.SERVER_NAME;
+import static oracle.kubernetes.operator.logging.MessageKeys.EXECUTE_MAKE_RIGHT_DOMAIN;
+import static oracle.kubernetes.operator.logging.MessageKeys.LOG_WAITING_COUNT;
+
 /**
  * Watches for changes to pods.
  */
@@ -305,6 +315,8 @@ public Step waitForDelete(V1Pod pod, Step next) {
 
   private abstract static class WaitForPodStatusStep extends WaitForReadyStep<V1Pod> {
 
+    public static final int RECHECK_DEBUG_COUNT = 10;
+
     private WaitForPodStatusStep(V1Pod pod, Step next) {
       super(pod, next);
     }
@@ -322,6 +334,67 @@ V1ObjectMeta getMetadata(V1Pod pod) {
     Step createReadAsyncStep(String name, String namespace, String domainUid, ResponseStep<V1Pod> responseStep) {
       return new CallBuilder().readPodAsync(name, namespace, domainUid, responseStep);
     }
+
+    protected DefaultResponseStep<V1Pod> resumeIfReady(Callback callback) {
+      return new DefaultResponseStep<>(getNext()) {
+        @Override
+        public NextAction onSuccess(Packet packet, CallResponse<V1Pod> callResponse) {
+
+          DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
+          String serverName = (String)packet.get(SERVER_NAME);
+          String resource = initialResource == null ? resourceName : getMetadata(initialResource).getName();
+          if ((info != null) && (callResponse != null)) {
+            Optional.ofNullable(callResponse.getResult()).ifPresent(result ->
+                    info.setServerPodFromEvent(getPodLabel(result), result));
+            if (onReadNotFoundForCachedResource(getServerPod(info, serverName), isNotFoundOnRead(callResponse))) {
+              LOGGER.fine(EXECUTE_MAKE_RIGHT_DOMAIN, serverName, callback.getRecheckCount());
+              removeCallback(resource, callback);
+              return doNext(NEXT_STEP_FACTORY.createMakeDomainRightStep(callback, info, getNext()), packet);
+            }
+          }
+
+          if (isReady(callResponse.getResult()) || callback.didResumeFiber()) {
+            callback.proceedFromWait(callResponse.getResult());
+            return null;
+          }
+
+          if (shouldWait()) {
+            if ((callback.getRecheckCount() % RECHECK_DEBUG_COUNT) == 0) {
+              LOGGER.fine(LOG_WAITING_COUNT,  serverName, callback.getRecheckCount());
+            }
+            // Watch backstop recheck count is less than or equal to the configured recheck count, delay.
+            return doDelay(createReadAndIfReadyCheckStep(callback), packet,
+                    getWatchBackstopRecheckDelaySeconds(), TimeUnit.SECONDS);
+          } else {
+            LOGGER.fine(EXECUTE_MAKE_RIGHT_DOMAIN, serverName, callback.getRecheckCount());
+            removeCallback(resource, callback);
+            // Watch backstop recheck count is more than configured recheck count, proceed to make-right step.
+            return doNext(NEXT_STEP_FACTORY.createMakeDomainRightStep(callback, info, getNext()), packet);
+          }
+        }
+
+        private String getPodLabel(V1Pod pod) {
+          return Optional.ofNullable(pod)
+                  .map(V1Pod::getMetadata)
+                  .map(V1ObjectMeta::getLabels)
+                  .map(m -> m.get(LabelConstants.SERVERNAME_LABEL))
+                  .orElse(null);
+        }
+
+        private V1Pod getServerPod(DomainPresenceInfo info, String serverName) {
+          return Optional.ofNullable(serverName).map(info::getServerPod).orElse(null);
+        }
+
+        private boolean isNotFoundOnRead(CallResponse callResponse) {
+          return callResponse.getResult() == null;
+        }
+
+        private boolean shouldWait() {
+          return callback.incrementAndGetRecheckCount() <= getWatchBackstopRecheckCount();
+        }
+      };
+    }
+
   }
 
   private class WaitForPodReadyStep extends WaitForPodStatusStep {
@@ -360,13 +433,25 @@ protected void removeCallback(String podName, Consumer<V1Pod> callback) {
     protected void logWaiting(String name) {
       LOGGER.fine(MessageKeys.WAITING_FOR_POD_READY, name);
     }
+
+    @Override
+    protected boolean onReadNotFoundForCachedResource(V1Pod cachedPod, boolean isNotFoundOnRead) {
+      // Return true if cached pod is not null but pod not found in explicit read, false otherwise.
+      return (cachedPod != null) && isNotFoundOnRead;
+    }
+
   }
 
   private class WaitForPodDeleteStep extends WaitForPodStatusStep {
     private WaitForPodDeleteStep(V1Pod pod, Step next) {
       super(pod, next);
     }
 
+    @Override
+    protected boolean onReadNotFoundForCachedResource(V1Pod cachedPod, boolean isNotFoundOnRead) {
+      return false;
+    }
+
     // A pod is considered deleted when reading its value from Kubernetes returns null.
     @Override
     protected boolean isReady(V1Pod result) {

operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java

@@ -194,17 +194,20 @@ class WatchTuning {
     public final int watchLifetime;
     public final int watchMinimumDelay;
     public final int watchBackstopRecheckDelay;
+    public final int watchBackstopRecheckCount;
 
     /**
      * Create watch tuning.
      * @param watchLifetime Watch lifetime
      * @param watchMinimumDelay Minimum delay before accepting new events to prevent hot loops
      * @param watchBackstopRecheckDelay Recheck delay for get while waiting for a status to backstop missed watch events
      */
-    public WatchTuning(int watchLifetime, int watchMinimumDelay, int watchBackstopRecheckDelay) {
+    public WatchTuning(int watchLifetime, int watchMinimumDelay, int watchBackstopRecheckDelay,
+                       int watchBackstopRecheckCount) {
       this.watchLifetime = watchLifetime;
       this.watchMinimumDelay = watchMinimumDelay;
       this.watchBackstopRecheckDelay = watchBackstopRecheckDelay;
+      this.watchBackstopRecheckCount = watchBackstopRecheckCount;
     }
 
     @Override

operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java

@@ -75,7 +75,8 @@ private void update() {
         new WatchTuning(
             (int) readTuningParameter("watchLifetime", 300),
             (int) readTuningParameter("watchMinimumDelay", 5),
-            (int) readTuningParameter("watchBackstopRecheckDelaySeconds", 5));
+            (int) readTuningParameter("watchBackstopRecheckDelaySeconds", 5),
+            (int) readTuningParameter("watchBackstopRecheckCount", 60));
 
     PodTuning pod =
         new PodTuning(