Merge branch 'cherry-pick-2f8e3f37' into 'release/4.1'

Cherry-pick branch 'failureInfo' into 'release/4.1'

See merge request weblogic-cloud/weblogic-kubernetes-operator!4551

Author: rjeberhard

documentation/domains/Domain.json

@@ -265,6 +265,10 @@
           "description": "Human-readable message indicating details about last transition.",
           "type": "string"
         },
+        "failureInfo": {
+          "description": "Details about the failure. This field will only be set when the condition type is Failed.",
+          "$ref": "#/definitions/DomainConditionFailureInfo"
+        },
         "status": {
           "description": "The status of the condition. Can be True, False, Unknown.",
           "type": "string"
@@ -275,6 +279,23 @@
         "status"
       ]
     },
+    "DomainConditionFailureInfo": {
+      "type": "object",
+      "properties": {
+        "introspectVersion": {
+          "description": "The introspectVersion set when the Failed condition occurred.",
+          "type": "string"
+        },
+        "introspectImage": {
+          "description": "The image used by the introspector when the Failed condition occurred.",
+          "type": "string"
+        },
+        "restartVersion": {
+          "description": "The restartVersion set when the Failed condition occurred.",
+          "type": "string"
+        }
+      }
+    },
     "DomainCreationImage": {
       "type": "object",
       "properties": {

documentation/domains/Domain.md

@@ -192,6 +192,7 @@ The current status of the operation of the WebLogic domain. Updated automaticall
 
 | Name | Type | Description |
 | --- | --- | --- |
+| `failureInfo` | [Domain Condition Failure Info](#domain-condition-failure-info) | Details about the failure. This field will only be set when the condition type is Failed. |
 | `lastTransitionTime` | DateTime | Last time the condition transitioned from one status to another. |
 | `message` | string | Human-readable message indicating details about last transition. |
 | `reason` | string | Unique, one-word, CamelCase reason for the condition's last transition. |
@@ -306,6 +307,14 @@ TopologySpreadConstraint specifies how to spread matching pods among the given t
 | `status` | string | The status of the condition. Can be True, False. |
 | `type` | string | The type of the condition. Valid types are Completed, Available, Failed, and Rolling. |
 
+### Domain Condition Failure Info
+
+| Name | Type | Description |
+| --- | --- | --- |
+| `introspectImage` | string | The image used by the introspector when the Failed condition occurred. |
+| `introspectVersion` | string | The introspectVersion set when the Failed condition occurred. |
+| `restartVersion` | string | The restartVersion set when the Failed condition occurred. |
+
 ### Server Health
 
 | Name | Type | Description |

kubernetes/crd/domain-crd.yaml

@@ -5,7 +5,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    weblogic.sha256: 01a3cdba6b993d2091f4c55f47ffb2a35debc51ee59d1de05ccd3735881d64f8
+    weblogic.sha256: 33268f8aa3d7ece940cdfa9e06d55685f8fb45de1d8f4f5f9186bd059d3b8da9
   name: domains.weblogic.oracle
 spec:
   group: weblogic.oracle
@@ -9745,6 +9745,23 @@ spec:
                       description: Human-readable message indicating details about
                         last transition.
                       type: string
+                    failureInfo:
+                      description: Details about the failure. This field will only
+                        be set when the condition type is Failed.
+                      type: object
+                      properties:
+                        introspectVersion:
+                          description: The introspectVersion set when the Failed condition
+                            occurred.
+                          type: string
+                        introspectImage:
+                          description: The image used by the introspector when the
+                            Failed condition occurred.
+                          type: string
+                        restartVersion:
+                          description: The restartVersion set when the Failed condition
+                            occurred.
+                          type: string
                     status:
                       description: The status of the condition. Can be True, False,
                         Unknown.

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

@@ -390,7 +390,7 @@ private boolean shouldContinue(MakeRightDomainOperation operation, DomainPresenc
     } else if (isDeleting(operation)) {
       return true;
     } else if (liveInfo.isDomainProcessingHalted(cachedInfo)) {
-      return false;
+      return liveInfo.isMustDomainProcessingRestart();
     } else if (isExplicitRecheckWithoutRetriableFailure(operation, liveInfo)
         || liveInfo.isDomainGenerationChanged(cachedInfo)) {
       return true;

operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java

@@ -1,4 +1,4 @@
-// Copyright (c) 2018, 2023, Oracle and/or its affiliates.
+// Copyright (c) 2018, 2024, Oracle and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 package oracle.kubernetes.operator;
@@ -577,7 +577,8 @@ List<EventData> createDomainEvents() {
     void addFailure(DomainStatus status, DomainCondition condition) {
       addFailureCondition(status, condition);
       if (hasReachedRetryLimit(status, condition)) {
-        addFailureCondition(status, new DomainCondition(FAILED).withReason(ABORTED).withMessage(getFatalMessage()));
+        addFailureCondition(status, new DomainCondition(FAILED).withReason(ABORTED)
+            .withFailureInfo(getDomain().getSpec()).withMessage(getFatalMessage()));
       }
     }
 
@@ -748,11 +749,14 @@ private void setStatusConditions(DomainStatus status) {
         newConditions.apply();
 
         if (isHasFailedPod()) {
-          addFailure(status, new DomainCondition(FAILED).withReason(SERVER_POD).withMessage(getPodFailedMessage()));
+          addFailure(status, new DomainCondition(FAILED).withReason(SERVER_POD)
+              .withFailureInfo(getDomain().getSpec()).withMessage(getPodFailedMessage()));
         } else if (hasPodNotRunningInTime()) {
-          addFailure(status, new DomainCondition(FAILED).withReason(SERVER_POD).withMessage(getPodNotRunningMessage()));
+          addFailure(status, new DomainCondition(FAILED).withReason(SERVER_POD)
+              .withFailureInfo(getDomain().getSpec()).withMessage(getPodNotRunningMessage()));
         } else if (hasPodNotReadyInTime()) {
-          addFailure(status, new DomainCondition(FAILED).withReason(SERVER_POD).withMessage(getPodNotReadyMessage()));
+          addFailure(status, new DomainCondition(FAILED).withReason(SERVER_POD)
+              .withFailureInfo(getDomain().getSpec()).withMessage(getPodNotReadyMessage()));
         } else {
           status.removeConditionsMatching(c -> c.hasType(FAILED) && SERVER_POD == c.getReason());
           if (newConditions.allIntendedServersReady() && !stillHasPodPendingRestart(status)) {
@@ -1586,7 +1590,8 @@ public FailureStatusUpdaterContext(Packet packet, DomainStatusUpdaterStep status
       @Override
       void modifyStatus(DomainStatus status) {
         removingReasons.forEach(status::markFailuresForRemoval);
-        addFailure(status, status.createAdjustedFailedCondition(reason, message, isInitializeDomainOnPV()));
+        addFailure(status, status.createAdjustedFailedCondition(reason, message, isInitializeDomainOnPV(),
+            getDomain().getSpec()));
         status.addCondition(new DomainCondition(COMPLETED).withStatus(false), isInitializeDomainOnPV());
         Optional.ofNullable(jobUid).ifPresent(status::setFailedIntrospectionUid);
         status.removeMarkedFailures();

operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java

@@ -1,4 +1,4 @@
-// Copyright (c) 2018, 2023, Oracle and/or its affiliates.
+// Copyright (c) 2018, 2024, Oracle and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 package oracle.kubernetes.operator.helpers;
@@ -49,6 +49,7 @@
 import oracle.kubernetes.weblogic.domain.model.ClusterResource;
 import oracle.kubernetes.weblogic.domain.model.ClusterSpec;
 import oracle.kubernetes.weblogic.domain.model.ClusterStatus;
+import oracle.kubernetes.weblogic.domain.model.DomainConditionFailureInfo;
 import oracle.kubernetes.weblogic.domain.model.DomainResource;
 import oracle.kubernetes.weblogic.domain.model.DomainSpec;
 import oracle.kubernetes.weblogic.domain.model.DomainStatus;
@@ -138,6 +139,10 @@ public boolean isDomainGenerationChanged(DomainPresenceInfo cachedInfo) {
     return getGeneration(getDomain()).compareTo(getGeneration(cachedInfo.getDomain())) > 0;
   }
 
+  public boolean isMustDomainProcessingRestart() {
+    return !failureInfoVersionsUnchanged();
+  }
+
   /**
    * Returns true if the state of the current domain presence info, when compared with the cached info for the same
    * domain, indicates that the make-right should not be run. The user has a number of options to resume processing
@@ -155,6 +160,21 @@ private boolean isDomainProcessingAborted() {
             .orElse(false);
   }
 
+  private DomainConditionFailureInfo findFailureInfo() {
+    return Optional.ofNullable(getDomain()).map(DomainResource::getStatus)
+        .map(DomainStatus::getFailureInfo).orElse(null);
+  }
+
+  private boolean failureInfoVersionsUnchanged() {
+    return Optional.ofNullable(findFailureInfo()).map(this::versionsUnchanged).orElse(true);
+  }
+
+  private boolean versionsUnchanged(DomainConditionFailureInfo failureInfo) {
+    return hasSameIntrospectVersion(failureInfo)
+        && hasSameRestartVersion(failureInfo)
+        && hasSameIntrospectImage(failureInfo);
+  }
+
   private boolean versionsUnchanged(DomainPresenceInfo cachedInfo) {
     return hasSameIntrospectVersion(cachedInfo)
         && hasSameRestartVersion(cachedInfo)
@@ -176,10 +196,14 @@ private boolean isNewerThan(DomainPresenceInfo cachedInfo) {
         || !KubernetesUtils.isFirstNewer(cachedInfo.getDomain().getMetadata(), getDomain().getMetadata());
   }
 
-  private boolean hasSameIntrospectVersion(DomainPresenceInfo cachedInfo) {
+  private boolean  hasSameIntrospectVersion(DomainPresenceInfo cachedInfo) {
     return Objects.equals(getIntrospectVersion(), cachedInfo.getIntrospectVersion());
   }
 
+  private boolean  hasSameIntrospectVersion(DomainConditionFailureInfo failureInfo) {
+    return Objects.equals(getIntrospectVersion(), failureInfo.getIntrospectVersion());
+  }
+
   private String getIntrospectVersion() {
     return Optional.ofNullable(getDomain())
         .map(DomainResource::getSpec)
@@ -191,6 +215,10 @@ private boolean hasSameRestartVersion(DomainPresenceInfo cachedInfo) {
     return Objects.equals(getRestartVersion(), cachedInfo.getRestartVersion());
   }
 
+  private boolean hasSameRestartVersion(DomainConditionFailureInfo failureInfo) {
+    return Objects.equals(getRestartVersion(), failureInfo.getRestartVersion());
+  }
+
   private String getRestartVersion() {
     return Optional.ofNullable(getDomain())
         .map(DomainResource::getRestartVersion)
@@ -201,6 +229,10 @@ private boolean hasSameIntrospectImage(DomainPresenceInfo cachedInfo) {
     return Objects.equals(getIntrospectImage(), cachedInfo.getIntrospectImage());
   }
 
+  private boolean hasSameIntrospectImage(DomainConditionFailureInfo failureInfo) {
+    return Objects.equals(getIntrospectImage(), failureInfo.getIntrospectImage());
+  }
+
   private String getIntrospectImage() {
     return Optional.ofNullable(getDomain())
         .map(DomainResource::getSpec)

operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java

@@ -1,4 +1,4 @@
-// Copyright (c) 2018, 2023, Oracle and/or its affiliates.
+// Copyright (c) 2018, 2024, Oracle and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 package oracle.kubernetes.operator.helpers;
@@ -233,7 +233,7 @@ private void addDomainFailureStatus(Packet packet, RequestParams requestParams,
 
   private void updateFailureStatus(
       @Nonnull DomainResource domain, RequestParams requestParams, ApiException apiException) {
-    DomainCondition condition = new DomainCondition(FAILED).withReason(KUBERNETES)
+    DomainCondition condition = new DomainCondition(FAILED).withFailureInfo(domain.getSpec()).withReason(KUBERNETES)
         .withMessage(createMessage(requestParams, apiException));
     addFailureStatus(domain, condition);
   }

operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainCondition.java

@@ -1,4 +1,4 @@
-// Copyright (c) 2017, 2023, Oracle and/or its affiliates.
+// Copyright (c) 2017, 2024, Oracle and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 package oracle.kubernetes.weblogic.domain.model;
@@ -46,6 +46,11 @@ public class DomainCondition implements Comparable<DomainCondition>, PatchableCo
   @Expose
   private DomainFailureReason reason;
 
+  @Description("Details about the failure. This field will only be set when the condition type is Failed.")
+  @SerializedName("failureInfo")
+  @Expose
+  private DomainConditionFailureInfo failureInfo;
+
   @Description("The status of the condition. Can be True, False, Unknown.")
   @SerializedName("status")
   @Expose
@@ -76,6 +81,7 @@ public DomainCondition(@Nonnull DomainConditionType conditionType) {
     this.status = other.status;
     this.markedForDeletion = other.markedForDeletion;
     this.severity = other.severity;
+    this.failureInfo = other.failureInfo;
   }
 
   /**
@@ -198,6 +204,24 @@ public DomainCondition withStatus(boolean status) {
     return type;
   }
 
+  public DomainConditionFailureInfo getFailureInfo() {
+    return failureInfo;
+  }
+
+  public void setFailureInfo(DomainConditionFailureInfo failureInfo) {
+    this.failureInfo = failureInfo;
+  }
+
+  public DomainCondition withFailureInfo(DomainConditionFailureInfo failureInfo) {
+    setFailureInfo(failureInfo);
+    return this;
+  }
+
+  public DomainCondition withFailureInfo(DomainSpec spec) {
+    setFailureInfo(new DomainConditionFailureInfo(spec));
+    return this;
+  }
+
   /**
    * Set the severity for the current FAILED condition. This is not allowed for other types.
    * @param severity the new severity value
@@ -246,6 +270,7 @@ public String toString() {
     Optional.ofNullable(reason).ifPresent(r -> sb.append(" reason: ").append(r));
     Optional.ofNullable(severity).ifPresent(m -> sb.append(" severity: ").append(m));
     Optional.ofNullable(message).ifPresent(m -> sb.append(" message: ").append(m));
+    Optional.ofNullable(failureInfo).ifPresent(f -> sb.append(" failureInfo: ").append(f));
     return sb.toString();
   }
 
@@ -257,6 +282,7 @@ public int hashCode() {
         .append(type)
         .append(status)
         .append(severity)
+        .append(failureInfo)
         .toHashCode();
   }
 
@@ -274,6 +300,7 @@ public boolean equals(Object other) {
         .append(type, rhs.type)
         .append(status, rhs.status)
         .append(severity, rhs.severity)
+        .append(failureInfo, rhs.failureInfo)
         .isEquals();
   }
 
@@ -295,7 +322,8 @@ int compareTransitionTime(DomainCondition thatCondition) {
         .withStringField("status", DomainCondition::getStatus)
         .withEnumField("reason", DomainCondition::getReason)
         .withEnumField("type", DomainCondition::getType)
-        .withEnumField("severity", DomainCondition::getSeverity);
+        .withEnumField("severity", DomainCondition::getSeverity)
+        .withObjectField("failureInfo", DomainCondition::getFailureInfo, DomainConditionFailureInfo.getObjectPatch());
 
   static ObjectPatch<DomainCondition> getObjectPatch() {
     return conditionPatch;