Merge branch 'fix-k8s-api-network-restart-error-42' into 'release/4.2'

Introduce k8s network failure reason and remove previous network related error...

See merge request weblogic-cloud/weblogic-kubernetes-operator!4831

Author: rjeberhard

documentation/domains/Domain.json

@@ -238,6 +238,7 @@
             "DomainInvalid",
             "Introspection",
             "Kubernetes",
+            "KubernetesNetworkException",
             "ServerPod",
             "PersistentVolumeClaim",
             "ReplicasTooHigh",

kubernetes/crd/domain-crd.yaml

@@ -5,7 +5,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    weblogic.sha256: 3cad99b24fb84de65dc38d7734ce269fb7058c3b6aed32b85ef590e142921635
+    weblogic.sha256: 8cd5a2176fe99b104c82048d750d42f1130341bdfdba825493bc64de45025424
   name: domains.weblogic.oracle
 spec:
   group: weblogic.oracle
@@ -10322,6 +10322,7 @@ spec:
                       - DomainInvalid
                       - Introspection
                       - Kubernetes
+                      - KubernetesNetworkException
                       - ServerPod
                       - PersistentVolumeClaim
                       - ReplicasTooHigh

operator/src/main/java/oracle/kubernetes/operator/StuckPodProcessing.java

@@ -64,6 +64,7 @@ public PodListProcessing(String namespace, OffsetDateTime dateTime) {
 
     @Override
     public NextAction onSuccess(Packet packet, CallResponse<V1PodList> callResponse) {
+      clearExistingKubernetesNetworkException(packet);
       callResponse.getResult().getItems().stream()
             .filter(pod -> isStuck(pod, now))
             .forEach(pod -> addStuckPodToPacket(packet, pod));
@@ -156,6 +157,7 @@ public ForcedDeleteResponseStep(String name, String namespace, String domainUID)
     @Override
     @SuppressWarnings("try")
     public NextAction onSuccess(Packet packet, CallResponse<Object> callResponse) {
+      clearExistingKubernetesNetworkException(packet);
       try (ThreadLoggingContext ignored =
                ThreadLoggingContext.setThreadContext().namespace(namespace).domainUid(domainUID)) {
         LOGGER.info(POD_FORCE_DELETED, name, namespace);

operator/src/main/java/oracle/kubernetes/operator/WaitForReadyStep.java

@@ -254,6 +254,7 @@ static class MakeRightDomainStep<V> extends DefaultResponseStep<V> {
 
     @Override
     public NextAction onSuccess(Packet packet, CallResponse<V> callResponse) {
+      clearExistingKubernetesNetworkException(packet);
       MakeRightDomainOperation makeRightDomainOperation =
               (MakeRightDomainOperation)packet.get(MAKE_RIGHT_DOMAIN_OPERATION);
       if (makeRightDomainOperation != null) {

operator/src/main/java/oracle/kubernetes/operator/WebhookMain.java

@@ -212,6 +212,7 @@ private class  CrdPresenceResponseStep<L extends KubernetesListObject> extends D
 
     @Override
     public NextAction onSuccess(Packet packet, CallResponse<L> callResponse) {
+      clearExistingKubernetesNetworkException(packet);
       warnedOfCrdAbsence = false;
       crdPresenceCheckCount.set(0);
       return super.onSuccess(packet, callResponse);

operator/src/main/java/oracle/kubernetes/operator/helpers/PersistentVolumeClaimHelper.java

@@ -144,6 +144,7 @@ private NextAction updateDomainStatus(Packet packet, CallResponse<V1PersistentVo
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PersistentVolumeClaim> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         logPersistentVolumeClaimCreated(messageKey);
         addPersistentVolumeClaimToRecord(callResponse.getResult());
         return doNext(packet);
@@ -164,6 +165,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1PersistentVolumeClaim>
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PersistentVolumeClaim> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
         V1PersistentVolumeClaim persistentVolumeClaim = callResponse.getResult();
 

operator/src/main/java/oracle/kubernetes/operator/helpers/PersistentVolumeHelper.java

@@ -133,6 +133,7 @@ private NextAction updateDomainStatus(Packet packet, CallResponse<V1PersistentVo
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PersistentVolume> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         logPersistentVolumeCreated(messageKey);
         return doNext(packet);
       }
@@ -152,6 +153,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1PersistentVolume> call
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PersistentVolume> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
         V1PersistentVolume persistentVolume = callResponse.getResult();
         if (persistentVolume == null) {

operator/src/main/java/oracle/kubernetes/operator/helpers/PodDisruptionBudgetHelper.java

@@ -113,6 +113,7 @@ private NextAction updateDomainStatus(Packet packet, CallResponse<V1PodDisruptio
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PodDisruptionBudget> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         logPodDisruptionBudgetCreated(messageKey);
         addPodDisruptionBudgetToRecord(callResponse.getResult());
         return doNext(packet);
@@ -133,6 +134,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1PodDisruptionBudget> c
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PodDisruptionBudget> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         V1PodDisruptionBudget podDisruptionBudget = callResponse.getResult();
         if (podDisruptionBudget == null) {
           removePodDisruptionBudgetFromRecord();
@@ -157,6 +159,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1PodDisruptionBudget> c
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PodDisruptionBudget> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         logPodDisruptionBudgetPatched();
         return doNext(packet);
       }

operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java

@@ -3,6 +3,8 @@
 
 package oracle.kubernetes.operator.helpers;
 
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
 import java.util.Collections;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -25,6 +27,7 @@
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
 import oracle.kubernetes.weblogic.domain.model.DomainCondition;
+import oracle.kubernetes.weblogic.domain.model.DomainFailureReason;
 import oracle.kubernetes.weblogic.domain.model.DomainResource;
 
 import static oracle.kubernetes.common.CommonConstants.CRD;
@@ -37,6 +40,7 @@
 import static oracle.kubernetes.operator.calls.AsyncRequestStep.accessContinue;
 import static oracle.kubernetes.weblogic.domain.model.DomainConditionType.FAILED;
 import static oracle.kubernetes.weblogic.domain.model.DomainFailureReason.KUBERNETES;
+import static oracle.kubernetes.weblogic.domain.model.DomainFailureReason.KUBERNETES_NETWORK_EXCEPTION;
 
 /**
  * Step to receive response of Kubernetes API server call.
@@ -80,6 +84,20 @@ public final void setPrevious(Step previousStep) {
     this.previousStep = previousStep;
   }
 
+  /**
+   * Clear out any existing Kubernetes network exception (ConnectException and SocketTimeoutException).
+   *
+   * @param packet packet
+   */
+  public static void clearExistingKubernetesNetworkException(Packet packet) {
+    Optional.ofNullable(packet.getSpi(DomainPresenceInfo.class))
+        .map(DomainPresenceInfo::getDomain)
+        .map(DomainResource::getStatus)
+        .ifPresent(status -> status.removeConditionsMatching(
+            c -> c.hasType(FAILED) && KUBERNETES_NETWORK_EXCEPTION == c.getReason()));
+  }
+
+
   @Override
   public final NextAction apply(Packet packet) {
     NextAction nextAction = getActionForCallResponse(packet);
@@ -233,7 +251,16 @@ private void addDomainFailureStatus(Packet packet, RequestParams requestParams,
 
   private void updateFailureStatus(
       @Nonnull DomainResource domain, RequestParams requestParams, ApiException apiException) {
-    DomainCondition condition = new DomainCondition(FAILED).withFailureInfo(domain.getSpec()).withReason(KUBERNETES)
+    DomainFailureReason reason = KUBERNETES;
+    if (apiException != null) {
+      LOGGER.fine("updateFailureStatus: apiException: " + apiException.getCause());
+      LOGGER.fine("updateFailureStatus: status code: " + apiException.getCode());
+    }
+    if (apiException != null && (apiException.getCause() instanceof ConnectException
+        || apiException.getCause() instanceof SocketTimeoutException)) {
+      reason = DomainFailureReason.KUBERNETES_NETWORK_EXCEPTION;
+    }
+    DomainCondition condition = new DomainCondition(FAILED).withFailureInfo(domain.getSpec()).withReason(reason)
         .withMessage(createMessage(requestParams, apiException));
     addFailureStatus(domain, condition);
   }

operator/src/main/java/oracle/kubernetes/operator/helpers/SecretHelper.java

@@ -102,6 +102,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1Secret> callResponse)
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1Secret> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         V1Secret secret = callResponse.getResult();
         packet.getSpi(DomainPresenceInfo.class).setWebLogicCredentialsSecret(secret);
         insertAuthorizationSource(packet, secret);