update restarting doc, delete shutdown marker during startup

Author: pfmackin

docs-source/content/userguide/managing-domains/domain-lifecycle/restarting.md

@@ -199,3 +199,10 @@ If you've created a new image that is not rolling compatible, and you've changed
     the image, or the Kubernetes resources that register your domain with the operator.  For example, your servers are caching information from an external database and you've modified the contents of the database.
 
     In these cases, you must manually initiate a restart.
+
+* **Managed Coherence Server safe shutdown**.
+
+    If the domain is configued to use Coherence Clusters, then the Operator will automatically wait until the Coherence HAStatus MBean attribute
+    indicates that it is safe to shutdown the server.  If, for some reason, the Operator is not able to access the Coherence MBean then the server
+    will not be shutdown until the domain timeoutSeconds expires.  To minimize any possibility of cache data loss, you should increase
+    the timeoutSeconds value to a large number, for example: 15 minutes.

integration-tests/src/test/java/oracle/kubernetes/operator/ITCoherenceTests.java

@@ -57,8 +57,8 @@ public static void staticPrepare() throws Exception {
   @AfterClass
   public static void staticUnPrepare() throws Exception {
     if (!QUICKTEST) {
-      tearDown(new Object() {}.getClass().getEnclosingClass().getSimpleName());
       operator1.destroy();
+      tearDown(new Object() {}.getClass().getEnclosingClass().getSimpleName());
     }
   }
 
@@ -193,7 +193,7 @@ private void restartDomainByChangingEnvProperty() throws Exception {
     // sleep and iterations
     //
     setWaitTimePod(2);
-    setMaxIterationsPod(125);
+    setMaxIterationsPod(150);
 
     domain.verifyDomainServerPodRestart(
         "\"-Dweblogic.StdoutDebugEnabled=false\"", "\"-Dweblogic.StdoutDebugEnabled=true\"");

operator/src/main/resources/scripts/startNodeManager.sh

@@ -110,6 +110,7 @@ else
   export SHUTDOWN_MARKER_FILE="${serverLogHome}/${SERVER_NAME}.shutdown"
   serverOutOption="-Dweblogic.Stdout=${SERVER_OUT_FILE}"
   createFolder "${serverLogHome}"
+  rm ${SHUTDOWN_MARKER_FILE}
 fi
 
 

operator/src/main/resources/scripts/startServer.sh

@@ -238,6 +238,8 @@ copySitCfg /weblogic-operator/introspector ${DOMAIN_HOME}/optconfig/jms
 copySitCfg /weblogic-operator/introspector ${DOMAIN_HOME}/optconfig/jdbc        'Sit-Cfg-JDBC--'
 copySitCfg /weblogic-operator/introspector ${DOMAIN_HOME}/optconfig/diagnostics 'Sit-Cfg-WLDF--'
 
+
+
 if [ "${MOCK_WLS}" == 'true' ]; then
   mockWLS
   waitForShutdownMarker