Backport 4250 and 4260 to release/4.0

Author: rjeberhard

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

@@ -188,9 +188,9 @@ static void cleanupNamespace(String namespace) {
   }
 
   private static void registerStatusUpdater(
-        String ns, String domainUid, ScheduledFuture<?> future) {
+      String ns, String domainUid, ScheduledFuture<?> future) {
     ScheduledFuture<?> existing =
-          statusUpdaters.computeIfAbsent(ns, k -> new ConcurrentHashMap<>()).put(domainUid, future);
+        statusUpdaters.computeIfAbsent(ns, k -> new ConcurrentHashMap<>()).put(domainUid, future);
     if (existing != null) {
       existing.cancel(false);
     }
@@ -293,8 +293,8 @@ private static void processServerEvent(CoreV1Event event) {
     }
 
     Optional.ofNullable(domains.get(event.getMetadata().getNamespace()))
-          .map(m -> m.get(domainUid))
-          .ifPresent(info -> info.updateLastKnownServerStatus(serverName, status));
+        .map(m -> m.get(domainUid))
+        .ifPresent(info -> info.updateLastKnownServerStatus(serverName, status));
   }
 
   /**
@@ -337,9 +337,9 @@ private void onDeleteEvent(@Nonnull String kind, @Nonnull String name, CoreV1Eve
 
   private static String getReadinessStatus(CoreV1Event event) {
     return Optional.ofNullable(event.getMessage())
-          .filter(m -> m.contains(WebLogicConstants.READINESS_PROBE_NOT_READY_STATE))
-          .map(m -> m.substring(m.lastIndexOf(':') + 1).trim())
-          .orElse(null);
+        .filter(m -> m.contains(WebLogicConstants.READINESS_PROBE_NOT_READY_STATE))
+        .map(m -> m.substring(m.lastIndexOf(':') + 1).trim())
+        .orElse(null);
   }
 
   // pre-conditions: DomainPresenceInfo SPI
@@ -379,10 +379,14 @@ private boolean shouldContinue(MakeRightDomainOperation operation, DomainPresenc
     final DomainPresenceInfo cachedInfo = getExistingDomainPresenceInfo(liveInfo);
     if (isNewDomain(cachedInfo)) {
       return true;
-    } else if (liveInfo.isFromOutOfDateEvent(operation, cachedInfo)
-        || liveInfo.isDomainProcessingHalted(cachedInfo)) {
+    } else if (liveInfo.isFromOutOfDateEvent(operation, cachedInfo)) {
+      return false;
+    } else if (isDeleting(operation)) {
+      return true;
+    } else if (liveInfo.isDomainProcessingHalted(cachedInfo)) {
       return false;
-    } else if (operation.isExplicitRecheck() || liveInfo.isDomainGenerationChanged(cachedInfo)) {
+    } else if (isExplicitRecheckWithoutRetriableFailure(operation, liveInfo)
+        || liveInfo.isDomainGenerationChanged(cachedInfo)) {
       return true;
     } else {
       cachedInfo.setDomain(liveInfo.getDomain());
@@ -392,7 +396,7 @@ private boolean shouldContinue(MakeRightDomainOperation operation, DomainPresenc
 
   private boolean shouldContinue(MakeRightClusterOperation operation, ClusterPresenceInfo liveInfo) {
     final ClusterPresenceInfo cachedInfo = getExistingClusterPresenceInfo(liveInfo);
-    if (hasDeletedClusterEventData(operation)) {
+    if (isDeleting(operation)) {
       return findClusterPresenceInfo(liveInfo.getNamespace(), liveInfo.getResourceName());
     } else if (isNewCluster(cachedInfo)) {
       return true;
@@ -406,6 +410,15 @@ private boolean shouldContinue(MakeRightClusterOperation operation, ClusterPrese
     }
   }
 
+  private boolean isExplicitRecheckWithoutRetriableFailure(
+      MakeRightDomainOperation operation, DomainPresenceInfo info) {
+    return operation.isExplicitRecheck() && !hasRetriableFailureNonRetryingOperation(operation, info);
+  }
+
+  private boolean hasRetriableFailureNonRetryingOperation(MakeRightDomainOperation operation, DomainPresenceInfo info) {
+    return info.hasRetriableFailure() && !operation.isRetryOnFailure();
+  }
+
   private boolean isNewDomain(DomainPresenceInfo cachedInfo) {
     return Optional.ofNullable(cachedInfo).map(DomainPresenceInfo::getDomain).orElse(null) == null;
   }
@@ -418,11 +431,15 @@ private boolean findClusterPresenceInfo(String namespace, String clusterName) {
     return Optional.ofNullable(clusters.get(namespace)).orElse(Collections.emptyMap()).get(clusterName) != null;
   }
 
-  private boolean hasDeletedClusterEventData(MakeRightClusterOperation operation) {
+  private boolean isDeleting(MakeRightClusterOperation operation) {
     return EventItem.CLUSTER_DELETED == getEventItem(operation);
   }
 
-  private EventItem getEventItem(MakeRightClusterOperation operation) {
+  private boolean isDeleting(MakeRightDomainOperation operation) {
+    return operation.isDeleting() || EventItem.DOMAIN_DELETED == getEventItem(operation);
+  }
+
+  private EventItem getEventItem(MakeRightOperation operation) {
     return Optional.ofNullable(operation.getEventData()).map(EventData::getItem).orElse(null);
   }
 
@@ -454,8 +471,8 @@ public void scheduleDomainStatusUpdates(DomainPresenceInfo info) {
   @Override
   public void registerDomainPresenceInfo(DomainPresenceInfo info) {
     domains
-          .computeIfAbsent(info.getNamespace(), k -> new ConcurrentHashMap<>())
-          .put(info.getDomainUid(), info);
+        .computeIfAbsent(info.getNamespace(), k -> new ConcurrentHashMap<>())
+        .put(info.getDomainUid(), info);
   }
 
   @Override
@@ -522,22 +539,22 @@ public void reportSuspendedFibers() {
     if (LOGGER.isFineEnabled()) {
       BiConsumer<String, FiberGate> consumer =
           (namespace, gate) -> gate.getCurrentFibers().forEach(
-            (key, fiber) -> Optional.ofNullable(fiber.getSuspendedStep()).ifPresent(suspendedStep -> {
-              try (ThreadLoggingContext ignored
-                  = setThreadContext().namespace(namespace).domainUid(getDomainUid(fiber))) {
-                LOGGER.fine("Fiber is SUSPENDED at " + suspendedStep.getResourceName());
-              }
-            }));
+              (key, fiber) -> Optional.ofNullable(fiber.getSuspendedStep()).ifPresent(suspendedStep -> {
+                try (ThreadLoggingContext ignored
+                         = setThreadContext().namespace(namespace).domainUid(getDomainUid(fiber))) {
+                  LOGGER.fine("Fiber is SUSPENDED at " + suspendedStep.getResourceName());
+                }
+              }));
       makeRightFiberGates.forEach(consumer);
       statusFiberGates.forEach(consumer);
     }
   }
 
   private String getDomainUid(Fiber fiber) {
     return Optional.ofNullable(fiber)
-          .map(Fiber::getPacket)
-          .map(p -> p.getSpi(DomainPresenceInfo.class))
-          .map(DomainPresenceInfo::getDomainUid).orElse("");
+        .map(Fiber::getPacket)
+        .map(p -> p.getSpi(DomainPresenceInfo.class))
+        .map(DomainPresenceInfo::getDomainUid).orElse("");
   }
 
   /**
@@ -622,10 +639,9 @@ private void processIntrospectorJobPodWatch(@Nonnull V1Pod pod, String watchType
   @Override
   public void updateDomainStatus(@Nonnull V1Pod pod, DomainPresenceInfo info) {
     Optional.ofNullable(IntrospectionStatus.createStatusUpdateSteps(pod))
-          .ifPresent(steps -> delegate.runSteps(new Packet().with(info), steps, null));
+        .ifPresent(steps -> delegate.runSteps(new Packet().with(info), steps, null));
   }
 
-
   /* Recently, we've seen a number of intermittent bugs where K8s reports
    * outdated watch events.  There seem to be two main cases: 1) a DELETED
    * event for a resource that was deleted, but has since been recreated, and 2)
@@ -677,7 +693,7 @@ public void dispatchPodDisruptionBudgetWatch(Watch.Response<V1PodDisruptionBudge
     }
 
     DomainPresenceInfo info =
-            getExistingDomainPresenceInfo(getPDBNamespace(pdb), domainUid);
+        getExistingDomainPresenceInfo(getPDBNamespace(pdb), domainUid);
     if (info == null) {
       return;
     }
@@ -718,7 +734,7 @@ public void dispatchConfigMapWatch(Watch.Response<V1ConfigMap> item) {
         case DELETED:
           delegate.runSteps(
               ConfigMapHelper.createScriptConfigMapStep(
-                    c.getMetadata().getNamespace(), productVersion));
+                  c.getMetadata().getNamespace(), productVersion));
           break;
 
         case ERROR:
@@ -993,7 +1009,7 @@ private void reportFailure(Throwable throwable) {
         logThrowable(throwable);
         runFailureSteps(throwable);
       }
-  
+
       private void runFailureSteps(Throwable throwable) {
         gate.startNewFiberIfCurrentFiberMatches(
             ((DomainPresenceInfo)presenceInfo).getDomainUid(),
@@ -1042,7 +1058,7 @@ private void scheduleRetry(@Nonnull DomainPresenceInfo domainPresenceInfo) {
       final MakeRightDomainOperation retry = operation.createRetry(domainPresenceInfo);
       gate.getExecutor().schedule(retry::execute, delayUntilNextRetry(domainPresenceInfo), TimeUnit.SECONDS);
     }
-    
+
     private long delayUntilNextRetry(@Nonnull DomainPresenceInfo domainPresenceInfo) {
       final OffsetDateTime nextRetryTime = domainPresenceInfo.getDomain().getNextRetryTime();
       final Duration interval = Duration.between(SystemClock.now(), nextRetryTime);

operator/src/main/java/oracle/kubernetes/operator/DomainResourcesValidation.java

@@ -273,8 +273,7 @@ private boolean isStranded(DomainPresenceInfo dpi) {
   private static void removeStrandedDomainPresenceInfo(DomainProcessor dp, DomainPresenceInfo info) {
     info.setDeleting(true);
     info.setPopulated(true);
-    dp.createMakeRightOperation(info).withExplicitRecheck().forDeletion().withEventData(new EventData(
-        EventItem.DOMAIN_DELETED)).execute();
+    dp.createMakeRightOperation(info).withExplicitRecheck().forDeletion().execute();
   }
 
   private Stream<DomainPresenceInfo> getActiveDomainPresenceInfos() {
@@ -290,9 +289,10 @@ private void activateDomain(DomainProcessor dp, DomainPresenceInfo info) {
     EventItem eventItem = getEventItem(info);
     MakeRightDomainOperation makeRight = dp.createMakeRightOperation(info).withExplicitRecheck();
     if (eventItem != null) {
-      makeRight.withEventData(new EventData(eventItem)).interrupt();
+      makeRight.withEventData(new EventData(eventItem)).interrupt().execute();
+    } else if (!info.hasRetriableFailure()) {
+      makeRight.execute();
     }
-    makeRight.execute();
   }
 
   private EventItem getEventItem(DomainPresenceInfo info) {

operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java

@@ -200,7 +200,7 @@ private boolean wasNotRolling() {
   /**
    * Creates an asynchronous step to initialize the domain status, if needed, to indicate that the operator has
    * seen the domain and is now working on it.
-   * @param hasEventData tue if the make right operation is associated with an event data
+   * @param hasEventData true if the make right operation is associated with an event.
    */
   public static Step createStatusInitializationStep(boolean hasEventData) {
     return new StatusInitializationStep(hasEventData);
@@ -593,7 +593,7 @@ void addDomainEvent(DomainCondition condition) {
   }
 
   public static class StatusInitializationStep extends DomainStatusUpdaterStep {
-    private boolean hasEventData;
+    private final boolean hasEventData;
 
     StatusInitializationStep(boolean hasEventData) {
       super();

operator/src/main/java/oracle/kubernetes/operator/MakeRightClusterOperation.java

@@ -10,6 +10,7 @@
  * Defines the operation to log a ClusterCreated/Changed/Deleted event.
  */
 public interface MakeRightClusterOperation extends MakeRightOperation<ClusterPresenceInfo> {
+
   /**
    * Set the event data that is associated with this operation.
    *
@@ -26,11 +27,4 @@ public interface MakeRightClusterOperation extends MakeRightOperation<ClusterPre
   MakeRightClusterOperation interrupt();
 
   MakeRightClusterOperation withExplicitRecheck();
-
-  /**
-   * Get the event data associated with this make-right operation.
-   *
-   * @return the event data.
-   */
-  EventHelper.EventData getEventData();
 }

operator/src/main/java/oracle/kubernetes/operator/MakeRightDomainOperation.java

@@ -44,8 +44,17 @@ public interface MakeRightDomainOperation extends MakeRightOperation<DomainPrese
    */
   MakeRightDomainOperation interrupt();
 
+  /**
+   * Modifies the factory to indicate that this is a retry operation on a retriable failure.
+   +
+   + @return the updated factory
+   */
+  MakeRightDomainOperation retryOnFailure();
+
   boolean isDeleting();
 
+  boolean isRetryOnFailure();
+
   void setInspectionRun();
 
   void setLiveInfo(@Nonnull DomainPresenceInfo info);
@@ -99,5 +108,4 @@ static Step createStepsToRerunWithIntrospection(Packet packet) {
   static Optional<MakeRightDomainOperation> fromPacket(Packet packet) {
     return Optional.ofNullable(packet.getValue(MAKE_RIGHT_DOMAIN_OPERATION));
   }
-
 }

operator/src/main/java/oracle/kubernetes/operator/MakeRightOperation.java

@@ -5,6 +5,7 @@
 
 import javax.annotation.Nonnull;
 
+import oracle.kubernetes.operator.helpers.EventHelper;
 import oracle.kubernetes.operator.helpers.ResourcePresenceInfo;
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.PacketComponent;
@@ -15,6 +16,7 @@
  * or to log a ClusterCreated/Changed/Deleted event.
  */
 public interface MakeRightOperation<T extends ResourcePresenceInfo> extends PacketComponent {
+
   void execute();
 
   @Nonnull
@@ -28,5 +30,12 @@ public interface MakeRightOperation<T extends ResourcePresenceInfo> extends Pack
 
   boolean hasEventData();
 
+  /**
+   * Get the event data associated with this make-right operation.
+   *
+   * @return the event data.
+   */
+  EventHelper.EventData getEventData();
+
   boolean isExplicitRecheck();
 }

operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java

@@ -1055,6 +1055,10 @@ public Collection<ClusterResource> getClusterResources() {
     return clusters.values();
   }
 
+  public boolean hasRetriableFailure() {
+    return Optional.ofNullable(getDomain()).map(DomainResource::hasRetriableFailure).orElse(false);
+  }
+
   /** Details about a specific managed server. */
   public static class ServerInfo {
     public final WlsServerConfig serverConfig;

operator/src/main/java/oracle/kubernetes/operator/makeright/MakeRightClusterOperationImpl.java

@@ -14,6 +14,7 @@
 import oracle.kubernetes.operator.helpers.ClusterPresenceInfo;
 import oracle.kubernetes.operator.helpers.EventHelper;
 import oracle.kubernetes.operator.helpers.EventHelper.ClusterResourceEventData;
+import oracle.kubernetes.operator.helpers.EventHelper.EventData;
 import oracle.kubernetes.operator.work.NextAction;
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
@@ -105,7 +106,7 @@ private boolean isDeleting() {
   }
 
   private EventHelper.EventItem getEventItem() {
-    return Optional.ofNullable(getEventData()).map(EventHelper.EventData::getItem).orElse(null);
+    return Optional.ofNullable(getEventData()).map(EventData::getItem).orElse(null);
   }
 
   @Override
@@ -123,11 +124,6 @@ public boolean isExplicitRecheck() {
     return explicitRecheck;
   }
 
-  @Override
-  public EventHelper.EventData getEventData() {
-    return eventData;
-  }
-
   class StartPlanStep extends Step {
 
     private final ClusterPresenceInfo info;