Async issues in Fiber and Watcher

Author: rjeberhard

operator/src/main/java/oracle/kubernetes/operator/Watcher.java

@@ -143,7 +143,7 @@ private void watchForEvents() {
             new WatchBuilder()
                 .withResourceVersion(resourceVersion.toString())
                 .withTimeoutSeconds(tuning.watchLifetime))) {
-      while (watch.hasNext()) {
+      while (hasNext(watch)) {
         Watch.Response<T> item = watch.next();
 
         if (isStopping()) {
@@ -164,6 +164,15 @@ private void watchForEvents() {
     }
   }
 
+  private boolean hasNext(WatchI<T> watch) {
+    try {
+      return watch.hasNext();
+    } catch (Throwable ex) {
+      // no-op on exception during hasNext
+    }
+    return false;
+  }
+
   /**
    * Initiates a watch by using the watch builder to request any updates for the specified watcher.
    *

operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java

@@ -190,7 +190,7 @@ public void onSuccess(
                         try {
                           cc.cancel();
                         } finally {
-                          LOGGER.info(
+                          LOGGER.fine(
                               MessageKeys.ASYNC_TIMEOUT,
                               requestParams.call,
                               requestParams.namespace,

operator/src/main/java/oracle/kubernetes/operator/work/Fiber.java

@@ -148,12 +148,12 @@ public void start(Step stepline, Packet packet, CompletionCallback completionCal
     this.na.invoke(stepline, packet);
     this.completionCallback = completionCallback;
 
-    if (LOGGER.isFineEnabled()) {
-      breadCrumbs = new ArrayList<>();
-      LOGGER.fine("{0} started", new Object[] {getName()});
-    }
-
     if (status.get() == NOT_COMPLETE) {
+      if (LOGGER.isFineEnabled()) {
+        breadCrumbs = new ArrayList<>();
+        LOGGER.fine("{0} started", new Object[] {getName()});
+      }
+
       owner.addRunnable(this);
     }
   }
@@ -262,7 +262,12 @@ public Fiber createChildFiber() {
         children = new ArrayList<>();
       }
       children.add(child);
-      addBreadCrumb(child);
+      if (status.get() == NOT_COMPLETE) {
+        addBreadCrumb(child);
+      } else {
+        // Race condition where child is created after parent is cancelled or done
+        child.status.set(CANCELLED);
+      }
     }
 
     return child;