Merge pull request #1041 from oracle/specify-no-version-samples

Add operator readiness probe and ensure that operator will not be alive or ready if initialization fails

Author: rjeberhard

docs/charts/index.yaml

@@ -1,25 +1,25 @@
 apiVersion: v1
 entries:
   weblogic-operator:
-  - created: 2019-04-04T14:48:45.073688-04:00
+  - created: 2019-04-19T18:49:45.158227-04:00
     description: Helm chart for configuring the WebLogic operator.
-    digest: c505f999cd43fdf9cd7a52d8e2db47a96e22e68f5116e47fb6dafee0bb31fa0e
+    digest: 83193bfdea8a9643428bc85cd154584bd80742d0f8c92b502039e939fd2884c4
     name: weblogic-operator
     urls:
     - https://oracle.github.io/weblogic-kubernetes-operator/charts/weblogic-operator-2.2.tgz
     version: "2.2"
-  - created: 2019-04-04T14:48:45.071125-04:00
+  - created: 2019-04-19T18:49:45.156092-04:00
     description: Helm chart for configuring the WebLogic operator.
     digest: 391e23c0969ada5f0cd2a088ddc6f11f237f57521801ed3925db2149a8437a0d
     name: weblogic-operator
     urls:
     - https://oracle.github.io/weblogic-kubernetes-operator/charts/weblogic-operator-2.1.tgz
     version: "2.1"
-  - created: 2019-04-04T14:48:45.06358-04:00
+  - created: 2019-04-19T18:49:45.153902-04:00
     description: Helm chart for configuring the WebLogic operator.
     digest: 298acda78ab73db6b7ba6f2752311bfa40c65874e03fb196b70976192211c1a5
     name: weblogic-operator
     urls:
     - https://oracle.github.io/weblogic-kubernetes-operator/charts/weblogic-operator-2.0.1.tgz
     version: 2.0.1
-generated: 2019-04-04T14:48:45.054609-04:00
+generated: 2019-04-19T18:49:45.149858-04:00

docs/charts/weblogic-operator-2.2.tgz

@@ -66,8 +66,15 @@ spec:
             command:
               - "bash"
               - "/operator/livenessProbe.sh"
-          initialDelaySeconds: 120
+          initialDelaySeconds: 20
           periodSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+              - "bash"
+              - "/operator/readinessProbe.sh"
+          initialDelaySeconds: 2
+          periodSeconds: 10
       {{- if .elkIntegrationEnabled }}
       - name: "logstash"
         image: {{ .logStashImage | quote }}

kubernetes/charts/weblogic-operator/templates/_operator-dep.tpl

@@ -232,13 +232,22 @@ protected ExtensionsV1beta1Deployment getExpectedWeblogicOperatorDeployment() {
                                                 .readOnly(true))
                                         .livenessProbe(
                                             newProbe()
-                                                .initialDelaySeconds(120)
+                                                .initialDelaySeconds(20)
                                                 .periodSeconds(5)
                                                 .exec(
                                                     newExecAction()
                                                         .addCommandItem("bash")
                                                         .addCommandItem(
-                                                            "/operator/livenessProbe.sh"))))
+                                                            "/operator/livenessProbe.sh")))
+                                        .readinessProbe(
+                                            newProbe()
+                                                .initialDelaySeconds(2)
+                                                .periodSeconds(10)
+                                                .exec(
+                                                    newExecAction()
+                                                        .addCommandItem("bash")
+                                                        .addCommandItem(
+                                                            "/operator/readinessProbe.sh"))))
                                 .addVolumesItem(
                                     newVolume()
                                         .name("weblogic-operator-cm-volume")

kubernetes/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesTestBase.java

@@ -174,9 +174,6 @@ public static void main(String[] args) {
       LOGGER.warning(MessageKeys.EXCEPTION, e);
     }
 
-    // start liveness thread
-    startLivenessThread();
-
     try {
       engine.getExecutor().execute(Main::begin);
 
@@ -226,6 +223,12 @@ private static void completeBegin() {
           .scheduleWithFixedDelay(
               recheckDomains(), recheckInterval, recheckInterval, TimeUnit.SECONDS);
 
+      // Wait until all other initialization is done before marking ready and
+      // starting liveness thread
+
+      // mark ready and start liveness thread
+      markReadyAndStartLivenessThread();
+
     } catch (Throwable e) {
       LOGGER.warning(MessageKeys.EXCEPTION, e);
     }
@@ -429,10 +432,16 @@ private static void stopRestServer() {
     RestServer.destroy();
   }
 
-  private static void startLivenessThread() {
-    LOGGER.info(MessageKeys.STARTING_LIVENESS_THREAD);
-    // every five seconds we need to update the last modified time on the liveness file
-    wrappedExecutorService.scheduleWithFixedDelay(new OperatorLiveness(), 5, 5, TimeUnit.SECONDS);
+  private static void markReadyAndStartLivenessThread() {
+    try {
+      OperatorReady.create();
+
+      LOGGER.info(MessageKeys.STARTING_LIVENESS_THREAD);
+      // every five seconds we need to update the last modified time on the liveness file
+      wrappedExecutorService.scheduleWithFixedDelay(new OperatorLiveness(), 5, 5, TimeUnit.SECONDS);
+    } catch (IOException io) {
+      LOGGER.severe(MessageKeys.EXCEPTION, io);
+    }
   }
 
   private static final Semaphore shutdownSignal = new Semaphore(0);

operator/src/main/java/oracle/kubernetes/operator/Main.java

@@ -0,0 +1,20 @@
+// Copyright 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Licensed under the Universal Permissive License v 1.0 as shown at
+// http://oss.oracle.com/licenses/upl.
+
+package oracle.kubernetes.operator;
+
+import java.io.File;
+import java.io.IOException;
+
+/** This task creates the "readiness" indicator so that Kubernetes knows the Operator is ready */
+public class OperatorReady {
+
+  private static final File readinessFile = new File("/operator/.ready");
+
+  public static void create() throws IOException {
+    if (!readinessFile.exists()) {
+      readinessFile.createNewFile();
+    }
+  }
+}

operator/src/main/java/oracle/kubernetes/operator/OperatorReady.java

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Copyright 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
+
+if [ ! -f /operator/.ready ]; then
+  exit 1;
+fi
+
+exit 0