Merge pull request #762 from oracle/diag-cm-threads

CPU burn bug observed with config map watch and lower logging levels for several entries.

Author: rjeberhard

operator/src/main/java/oracle/kubernetes/operator/ConfigMapWatcher.java

@@ -1,4 +1,4 @@
-// Copyright 2018, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Copyright 2018, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
 // Licensed under the Universal Permissive License v 1.0 as shown at
 // http://oss.oracle.com/licenses/upl.
 

operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java

@@ -1,4 +1,4 @@
-// Copyright 2017, 2018, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Copyright 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
 // Licensed under the Universal Permissive License v 1.0 as shown at
 // http://oss.oracle.com/licenses/upl.
 
@@ -7,6 +7,9 @@
 import java.io.IOException;
 import java.util.Map;
 import java.util.concurrent.ScheduledExecutorService;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
 
 public interface TuningParameters extends Map<String, String> {
 
@@ -47,6 +50,55 @@ public MainTuning(
       this.initialShortDelay = initialShortDelay;
       this.eventualLongDelay = eventualLongDelay;
     }
+
+    @Override
+    public String toString() {
+      return new ToStringBuilder(this)
+          .append("domainPresenceFailureRetrySeconds", domainPresenceFailureRetrySeconds)
+          .append("domainPresenceFailureRetryMaxCount", domainPresenceFailureRetryMaxCount)
+          .append("domainPresenceRecheckIntervalSeconds", domainPresenceRecheckIntervalSeconds)
+          .append("targetNamespaceRecheckIntervalSeconds", targetNamespaceRecheckIntervalSeconds)
+          .append("statusUpdateTimeoutSeconds", statusUpdateTimeoutSeconds)
+          .append("unchangedCountToDelayStatusRecheck", unchangedCountToDelayStatusRecheck)
+          .append("initialShortDelay", initialShortDelay)
+          .append("eventualLongDelay", eventualLongDelay)
+          .toString();
+    }
+
+    @Override
+    public int hashCode() {
+      return new HashCodeBuilder()
+          .append(domainPresenceFailureRetrySeconds)
+          .append(domainPresenceFailureRetryMaxCount)
+          .append(domainPresenceRecheckIntervalSeconds)
+          .append(targetNamespaceRecheckIntervalSeconds)
+          .append(statusUpdateTimeoutSeconds)
+          .append(unchangedCountToDelayStatusRecheck)
+          .append(initialShortDelay)
+          .append(eventualLongDelay)
+          .toHashCode();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (o == null) {
+        return false;
+      }
+      if (!(o instanceof MainTuning)) {
+        return false;
+      }
+      MainTuning mt = (MainTuning) o;
+      return new EqualsBuilder()
+          .append(domainPresenceFailureRetrySeconds, mt.domainPresenceFailureRetrySeconds)
+          .append(domainPresenceFailureRetryMaxCount, mt.domainPresenceFailureRetryMaxCount)
+          .append(domainPresenceRecheckIntervalSeconds, mt.domainPresenceRecheckIntervalSeconds)
+          .append(targetNamespaceRecheckIntervalSeconds, mt.targetNamespaceRecheckIntervalSeconds)
+          .append(statusUpdateTimeoutSeconds, mt.statusUpdateTimeoutSeconds)
+          .append(unchangedCountToDelayStatusRecheck, mt.unchangedCountToDelayStatusRecheck)
+          .append(initialShortDelay, mt.initialShortDelay)
+          .append(eventualLongDelay, mt.eventualLongDelay)
+          .isEquals();
+    }
   }
 
   public static class CallBuilderTuning {
@@ -59,13 +111,77 @@ public CallBuilderTuning(int callRequestLimit, int callMaxRetryCount, int callTi
       this.callMaxRetryCount = callMaxRetryCount;
       this.callTimeoutSeconds = callTimeoutSeconds;
     }
+
+    @Override
+    public String toString() {
+      return new ToStringBuilder(this)
+          .append("callRequestLimit", callRequestLimit)
+          .append("callMaxRetryCount", callMaxRetryCount)
+          .append("callTimeoutSeconds", callTimeoutSeconds)
+          .toString();
+    }
+
+    @Override
+    public int hashCode() {
+      return new HashCodeBuilder()
+          .append(callRequestLimit)
+          .append(callMaxRetryCount)
+          .append(callTimeoutSeconds)
+          .toHashCode();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (o == null) {
+        return false;
+      }
+      if (!(o instanceof CallBuilderTuning)) {
+        return false;
+      }
+      CallBuilderTuning cbt = (CallBuilderTuning) o;
+      return new EqualsBuilder()
+          .append(callRequestLimit, cbt.callRequestLimit)
+          .append(callMaxRetryCount, cbt.callMaxRetryCount)
+          .append(callTimeoutSeconds, cbt.callTimeoutSeconds)
+          .isEquals();
+    }
   }
 
   public static class WatchTuning {
     public final int watchLifetime;
+    public final int watchMinimumDelay;
 
-    public WatchTuning(int watchLifetime) {
+    public WatchTuning(int watchLifetime, int watchMinimumDelay) {
       this.watchLifetime = watchLifetime;
+      this.watchMinimumDelay = watchMinimumDelay;
+    }
+
+    @Override
+    public String toString() {
+      return new ToStringBuilder(this)
+          .append("watchLifetime", watchLifetime)
+          .append("watchMinimumDelay", watchMinimumDelay)
+          .toString();
+    }
+
+    @Override
+    public int hashCode() {
+      return new HashCodeBuilder().append(watchLifetime).append(watchMinimumDelay).toHashCode();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (o == null) {
+        return false;
+      }
+      if (!(o instanceof WatchTuning)) {
+        return false;
+      }
+      WatchTuning wt = (WatchTuning) o;
+      return new EqualsBuilder()
+          .append(watchLifetime, wt.watchLifetime)
+          .append(watchMinimumDelay, wt.watchMinimumDelay)
+          .isEquals();
     }
   }
 
@@ -91,6 +207,49 @@ public PodTuning(
       this.livenessProbeTimeoutSeconds = livenessProbeTimeoutSeconds;
       this.livenessProbePeriodSeconds = livenessProbePeriodSeconds;
     }
+
+    @Override
+    public String toString() {
+      return new ToStringBuilder(this)
+          .append("readinessProbeInitialDelaySeconds", readinessProbeInitialDelaySeconds)
+          .append("readinessProbeTimeoutSeconds", readinessProbeTimeoutSeconds)
+          .append("readinessProbePeriodSeconds", readinessProbePeriodSeconds)
+          .append("livenessProbeInitialDelaySeconds", livenessProbeInitialDelaySeconds)
+          .append("livenessProbeTimeoutSeconds", livenessProbeTimeoutSeconds)
+          .append("livenessProbePeriodSeconds", livenessProbePeriodSeconds)
+          .toString();
+    }
+
+    @Override
+    public int hashCode() {
+      return new HashCodeBuilder()
+          .append(readinessProbeInitialDelaySeconds)
+          .append(readinessProbeTimeoutSeconds)
+          .append(readinessProbePeriodSeconds)
+          .append(livenessProbeInitialDelaySeconds)
+          .append(livenessProbeTimeoutSeconds)
+          .append(livenessProbePeriodSeconds)
+          .toHashCode();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (o == null) {
+        return false;
+      }
+      if (!(o instanceof PodTuning)) {
+        return false;
+      }
+      PodTuning pt = (PodTuning) o;
+      return new EqualsBuilder()
+          .append(readinessProbeInitialDelaySeconds, pt.readinessProbeInitialDelaySeconds)
+          .append(readinessProbeTimeoutSeconds, pt.readinessProbeTimeoutSeconds)
+          .append(readinessProbePeriodSeconds, pt.readinessProbePeriodSeconds)
+          .append(livenessProbeInitialDelaySeconds, pt.livenessProbeInitialDelaySeconds)
+          .append(livenessProbeTimeoutSeconds, pt.livenessProbeTimeoutSeconds)
+          .append(livenessProbePeriodSeconds, pt.livenessProbePeriodSeconds)
+          .isEquals();
+    }
   }
 
   public MainTuning getMainTuning();

operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java

@@ -1,4 +1,4 @@
-// Copyright 2017, 2018, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Copyright 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
 // Licensed under the Universal Permissive License v 1.0 as shown at
 // http://oss.oracle.com/licenses/upl.
 
@@ -47,8 +47,6 @@ private static void updateTuningParameters() {
   }
 
   private void update() {
-    LOGGER.info(MessageKeys.TUNING_PARAMETERS);
-
     MainTuning main =
         new MainTuning(
             (int) readTuningParameter("domainPresenceFailureRetrySeconds", 10),
@@ -66,7 +64,10 @@ private void update() {
             (int) readTuningParameter("callMaxRetryCount", 5),
             (int) readTuningParameter("callTimeoutSeconds", 10));
 
-    WatchTuning watch = new WatchTuning((int) readTuningParameter("watchLifetime", 300));
+    WatchTuning watch =
+        new WatchTuning(
+            (int) readTuningParameter("watchLifetime", 300),
+            (int) readTuningParameter("watchMinimumDelay", 5));
 
     PodTuning pod =
         new PodTuning(
@@ -79,6 +80,12 @@ private void update() {
 
     lock.writeLock().lock();
     try {
+      if (!main.equals(this.main)
+          || !callBuilder.equals(this.callBuilder)
+          || !watch.equals(this.watch)
+          || !pod.equals(this.pod)) {
+        LOGGER.info(MessageKeys.TUNING_PARAMETERS);
+      }
       this.main = main;
       this.callBuilder = callBuilder;
       this.watch = watch;

operator/src/main/java/oracle/kubernetes/operator/Watcher.java

@@ -38,6 +38,7 @@ abstract class Watcher<T> {
   private AtomicBoolean stopping;
   private WatchListener<T> listener;
   private Thread thread = null;
+  private long lastInitialize = 0;
 
   /**
    * Constructs a watcher without specifying a listener. Needed when the listener is the watch
@@ -120,6 +121,18 @@ protected boolean isStopping() {
   }
 
   private void watchForEvents() {
+    long now = System.currentTimeMillis();
+    long delay = (tuning.watchMinimumDelay * 1000) - (now - lastInitialize);
+    if (lastInitialize != 0 && delay > 0) {
+      try {
+        Thread.sleep(delay);
+      } catch (InterruptedException ex) {
+        LOGGER.warning(MessageKeys.EXCEPTION, ex);
+      }
+      lastInitialize = System.currentTimeMillis();
+    } else {
+      lastInitialize = now;
+    }
     try (WatchI<T> watch =
         initiateWatch(
             new WatchBuilder()

operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java

@@ -90,7 +90,7 @@ private V1ObjectMeta createMetadata() {
 
     private synchronized Map<String, String> loadScriptsFromClasspath() {
       Map<String, String> scripts = scriptReader.loadFilesFromClasspath();
-      LOGGER.info(MessageKeys.SCRIPT_LOADED, this.domainNamespace);
+      LOGGER.fine(MessageKeys.SCRIPT_LOADED, this.domainNamespace);
       return scripts;
     }
 

operator/src/main/java/oracle/kubernetes/operator/steps/ConfigMapAfterStep.java

@@ -1,4 +1,4 @@
-// Copyright 2017, 2018, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Copyright 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
 // Licensed under the Universal Permissive License v 1.0 as shown at
 // http://oss.oracle.com/licenses/upl.
 

operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java

@@ -1,4 +1,4 @@
-// Copyright 2018 Oracle Corporation and/or its affiliates.  All rights reserved.
+// Copyright 2018, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
 // Licensed under the Universal Permissive License v 1.0 as shown at
 // http://oss.oracle.com/licenses/upl.
 
@@ -41,7 +41,7 @@ public abstract class WatcherTestBase extends ThreadFactoryTestBase
 
   private int resourceVersion = INITIAL_RESOURCE_VERSION;
 
-  protected WatchTuning tuning = new WatchTuning(30);
+  protected WatchTuning tuning = new WatchTuning(30, 0);
 
   private V1ObjectMeta createMetaData() {
     return createMetaData("test", NAMESPACE);

operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java

@@ -290,7 +290,7 @@ static String getJobName() {
   }
 
   FiberTestSupport.StepFactory getStepFactory() {
-    return next -> JobHelper.createDomainIntrospectorJobStep(new WatchTuning(30), next);
+    return next -> JobHelper.createDomainIntrospectorJobStep(new WatchTuning(30, 0), next);
   }
 
   V1PodList createListPods() {