OWLS-90614 - Handle ApiException with 409 code returned from the replaceEvent API call (#2443)

* OWLS-90614 - Handle api exception with 409 code and "storage error" during replace event and other API calls.

* Changes to always create a new event after 409 error.

* Add async call name to the ASYNC_NO_RETRY log message.

Author: ankedia

operator/src/main/java/oracle/kubernetes/operator/calls/UnrecoverableErrorBuilder.java

@@ -28,6 +28,10 @@ public static <T> boolean isAsyncCallNotFoundFailure(CallResponse<T> callRespons
     return callResponse.isFailure() && isNotFound(callResponse.getE());
   }
 
+  public static <T> boolean isAsyncCallConflictFailure(CallResponse<T> callResponse) {
+    return callResponse.isFailure() && hasConflict(callResponse.getE());
+  }
+
   private static boolean isUnrecoverable(ApiException e) {
     return UnrecoverableErrorBuilderImpl.isUnrecoverable(e);
   }
@@ -36,6 +40,10 @@ private static boolean isNotFound(ApiException e) {
     return UnrecoverableErrorBuilderImpl.isNotFound(e);
   }
 
+  private static boolean hasConflict(ApiException e) {
+    return UnrecoverableErrorBuilderImpl.hasConflict(e);
+  }
+
   /**
    * Populate FailureStatusSource from a failed call response.
    * @param callResponse the failed call response

operator/src/main/java/oracle/kubernetes/operator/calls/unprocessable/UnrecoverableErrorBuilderImpl.java

@@ -69,6 +69,10 @@ public static boolean isNotFound(ApiException e) {
     return code == 404 || code == 410;
   }
 
+  public static boolean hasConflict(ApiException e) {
+    return e.getCode() == 409;
+  }
+
   /**
    * Create unrecoverable error builder.
    */

operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java

@@ -257,7 +257,8 @@ public NextAction onFailure(Packet packet, CallResponse<CoreV1Event> callRespons
         if (hasLoggedForbiddenNSWatchStoppedEvent(this, callResponse)) {
           return doNext(packet);
         }
-        if (UnrecoverableErrorBuilder.isAsyncCallNotFoundFailure(callResponse)) {
+        if (UnrecoverableErrorBuilder.isAsyncCallNotFoundFailure(callResponse)
+                || UnrecoverableErrorBuilder.isAsyncCallConflictFailure(callResponse)) {
           return doNext(Step.chain(createCreateEventCall(createEventModel(packet, eventData)), getNext()), packet);
         } else if (UnrecoverableErrorBuilder.isAsyncCallUnrecoverableFailure(callResponse)) {
           return onFailureNoRetry(packet, callResponse);

operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java

@@ -139,7 +139,8 @@ private NextAction doPotentialRetry(Step conflictStep, Packet packet, CallRespon
         .map(rs -> rs.doPotentialRetry(conflictStep, packet, callResponse.getStatusCode()))
         .orElseGet(() -> {
           LOGGER.fine(MessageKeys.ASYNC_NO_RETRY,
-              callResponse.getExceptionString(), callResponse.getStatusCode(), callResponse.getHeadersString());
+                  callResponse.getRequestParams().call,
+                  callResponse.getExceptionString(), callResponse.getStatusCode(), callResponse.getHeadersString());
           return null;
         });
   }

operator/src/main/resources/Operator.properties

@@ -60,7 +60,7 @@ WLSKO-0089=Pod {0} has failed
 WLSKO-0094=Async call {0} invoking: {1}, namespace: {2}, name: {3}, body: {4}, fieldSelector: {5}, labelSelector: {6}, resourceVersion: {7}
 WLSKO-0095=Async call {0} failed: {1}, code: {2}, headers {3} after invoking {4}, namespace: {5}, name: {6}, body: {7}, fieldSelector: {8}, labelSelector: {9}, resourceVersion: {10}, response body: {11}
 WLSKO-0096=Async call {0} invoking: {1} succeeded: {2}, code: {3}, headers {4}
-WLSKO-0097=Async call will not be retried: {0}, code: {1}, headers {2}
+WLSKO-0097=Async call {0} will not be retried: {1}, code: {2}, headers {3}
 WLSKO-0098=Async call {0} will be retried after delay: {1} ms: {2}, namespace: {3}, name: {4}
 WLSKO-0099=Async call {0} timed-out: {1}, namespace: {2}, name: {3}, body: {4}, fieldSelector: {5}, labelSelector: {6}, resourceVersion: {7}
 WLSKO-0101=Watch event: {0}, object: {1}

operator/src/test/java/oracle/kubernetes/operator/helpers/EventHelperTest.java

@@ -277,6 +277,22 @@ public void whenCreateEventTwice_fail404OnReplaceEvent_domainProcessingStartingE
         containsEventsWithCountOne(getEvents(testSupport), DOMAIN_PROCESSING_STARTING_EVENT, 2));
   }
 
+  @Test
+  public void whenCreateEventTwice_fail409OnReplaceEvent_domainProcessingStartingEventCreatedTwice() {
+    testSupport.runSteps(Step.chain(
+            createEventStep(new EventData(DOMAIN_PROCESSING_STARTING)),
+            createEventStep(new EventData(DOMAIN_PROCESSING_COMPLETED))));
+
+    dispatchAddedEventWatches();
+    CoreV1Event event = EventTestUtils.getEventWithReason(getEvents(testSupport), DOMAIN_PROCESSING_STARTING_EVENT);
+    testSupport.failOnReplace(EVENT, EventTestUtils.getName(event), NS, HTTP_CONFLICT);
+
+    testSupport.runSteps(Step.chain(createEventStep(new EventData(DOMAIN_PROCESSING_STARTING))));
+
+    assertThat("Found 2 DOMAIN_PROCESSING_STARTING events with count 1",
+            containsEventsWithCountOne(getEvents(testSupport), DOMAIN_PROCESSING_STARTING_EVENT, 2));
+  }
+
   @Test
   public void whenCreateEventTwice_fail410OnReplaceEvent_domainProcessingStartingEventCreatedTwice() {
     testSupport.runSteps(Step.chain(
@@ -819,8 +835,8 @@ public void whenNSWatchStoppedEventCreatedTwice_fail409OnReplace_eventCreatedOnc
 
     testSupport.runSteps(eventStep);
 
-    assertThat("Found 1 NAMESPACE_WATCHING_STOPPED event with expected count 2",
-        containsOneEventWithCount(getEvents(testSupport),
+    assertThat("Found 2 NAMESPACE_WATCHING_STOPPED event with expected count 1",
+            containsEventsWithCountOne(getEvents(testSupport),
             NAMESPACE_WATCHING_STOPPED_EVENT, 2), is(true));
   }