Add a log message when replace NamespaceWatchingStopped event hits 403 (#2323)

* log a message when replace NS_WATCHING_STOPPED event hits 403 too

Author: doxiao

operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java

@@ -195,9 +195,7 @@ public NextAction onSuccess(Packet packet, CallResponse<CoreV1Event> callRespons
 
       @Override
       public NextAction onFailure(Packet packet, CallResponse<CoreV1Event> callResponse) {
-        if (isForbiddenForNamespaceWatchingStoppedEvent(callResponse)) {
-          LOGGER.info(MessageKeys.CREATING_EVENT_FORBIDDEN,
-              eventData.eventItem.getReason(), eventData.getNamespace());
+        if (hasLoggedForbiddenNSWatchStoppedEvent(this, callResponse)) {
           return doNext(packet);
         }
 
@@ -250,6 +248,9 @@ public NextAction onSuccess(Packet packet, CallResponse<CoreV1Event> callRespons
       @Override
       public NextAction onFailure(Packet packet, CallResponse<CoreV1Event> callResponse) {
         restoreExistingEvent();
+        if (hasLoggedForbiddenNSWatchStoppedEvent(this, callResponse)) {
+          return doNext(packet);
+        }
         if (UnrecoverableErrorBuilder.isAsyncCallNotFoundFailure(callResponse)) {
           return doNext(Step.chain(createCreateEventCall(createEventModel(packet, eventData)), getNext()), packet);
         } else if (UnrecoverableErrorBuilder.isAsyncCallUnrecoverableFailure(callResponse)) {
@@ -278,6 +279,20 @@ private Step createEventRefreshStep(CoreV1Event event) {
       }
     }
 
+    private boolean isForbiddenForNSWatchStoppedEvent(
+        ResponseStep responseStep, CallResponse<CoreV1Event> callResponse) {
+      return responseStep.isForbidden(callResponse) && NAMESPACE_WATCHING_STOPPED == eventData.eventItem;
+    }
+
+    private boolean hasLoggedForbiddenNSWatchStoppedEvent(
+        ResponseStep responseStep, CallResponse<CoreV1Event> callResponse) {
+      if (isForbiddenForNSWatchStoppedEvent(responseStep, callResponse)) {
+        LOGGER.info(MessageKeys.CREATING_EVENT_FORBIDDEN, eventData.eventItem.getReason(), eventData.getNamespace());
+        return true;
+      }
+      return false;
+    }
+
     private static class ReadEventResponseStep extends ResponseStep<CoreV1Event> {
       ReadEventResponseStep(Step next) {
         super(next);

operator/src/main/resources/Operator.properties

@@ -131,7 +131,7 @@ WLSKO-0178=Operator cannot proceed, as the Custom Resource Definition for ''doma
 WLSKO-0179=Pod {0} in namespace {1} detected as stuck, and force-deleted
 WLSKO-0180=Creating event: {0}
 WLSKO-0181=Replacing event: {0}
-WLSKO-0182=Cannot create {0} event in namespace {1} due to an authorization error
+WLSKO-0182=Cannot create or replace {0} event in namespace {1} due to an authorization error
 WLSKO-0183=Creating Pod Disruption Budget for WebLogic domain with UID: {0}. Cluster name: {1}.
 WLSKO-0184=Existing Pod Disruption Budget is correct for WebLogic domain with UID: {0}. Cluster name: {1}.
 WLSKO-0185=Patching Pod Disruption Budget for WebLogic domain with UID: {0}. Cluster name: {1}.

operator/src/test/java/oracle/kubernetes/operator/helpers/EventHelperTest.java

@@ -667,6 +667,34 @@ public void whenNSWatchStoppedEventCreated_fail403OnCreate_foundExpectedLogMessa
     assertThat(logRecords, containsInfo(CREATING_EVENT_FORBIDDEN, NAMESPACE_WATCHING_STOPPED_EVENT, NS));
   }
 
+  @Test
+  public void whenNSWatchStoppedEventCreatedTwice_fail403OnReplace_eventCreatedOnce() {
+    testSupport.runSteps(Step.chain(createEventStep(new EventData(NAMESPACE_WATCHING_STOPPED))));
+
+    CoreV1Event event = EventTestUtils.getEventWithReason(getEvents(testSupport), NAMESPACE_WATCHING_STOPPED_EVENT);
+    dispatchAddedEventWatches();
+    testSupport.failOnReplace(EVENT, EventTestUtils.getName(event), NS, HTTP_FORBIDDEN);
+
+    testSupport.runSteps(Step.chain(createEventStep(new EventData(NAMESPACE_WATCHING_STOPPED))));
+
+    assertThat("Found 1 NAMESPACE_WATCHING_STOPPED event with expected count 1",
+        containsOneEventWithCount(getEvents(testSupport), NAMESPACE_WATCHING_STOPPED_EVENT, 1), is(true));
+  }
+
+  @Test
+  public void whenNSWatchStoppedEventCreatedTwice_fail403OnReplace_foundExpectedLogMessage() {
+    loggerControl.withLogLevel(Level.INFO).collectLogMessages(logRecords, CREATING_EVENT_FORBIDDEN);
+    testSupport.runSteps(Step.chain(createEventStep(new EventData(NAMESPACE_WATCHING_STOPPED))));
+
+    CoreV1Event event = EventTestUtils.getEventWithReason(getEvents(testSupport), NAMESPACE_WATCHING_STOPPED_EVENT);
+    dispatchAddedEventWatches();
+    testSupport.failOnReplace(EVENT, EventTestUtils.getName(event), NS, HTTP_FORBIDDEN);
+
+    testSupport.runSteps(Step.chain(createEventStep(new EventData(NAMESPACE_WATCHING_STOPPED))));
+
+    assertThat(logRecords, containsInfo(CREATING_EVENT_FORBIDDEN, NAMESPACE_WATCHING_STOPPED_EVENT, NS));
+  }
+
   @Test
   public void whenCreateEventStepCalledForNSWatchStartedEvent_eventCreatedWithExpectedMessage() {
     testSupport.runSteps(createEventStep(new EventData(NAMESPACE_WATCHING_STARTED).namespace(NS).resourceName(NS)));