Updating teardown for graceful shutdown procedure (#490)

* Updating teardown for graceful shutdown procedure

* Attempting to fix tests that involve restarts

Author: nwang92

splunk/common-files/entrypoint.sh

@@ -27,7 +27,10 @@ setup() {
 
 teardown() {
 	# Always run the stop command on termination
-	${SPLUNK_HOME}/bin/splunk stop 2>/dev/null || true
+	if [ `whoami` != "${SPLUNK_USER}" ]; then
+		RUN_AS_SPLUNK="sudo -u ${SPLUNK_USER}"
+	fi
+	${RUN_AS_SPLUNK} ${SPLUNK_HOME}/bin/splunk stop || true
 }
 
 trap teardown SIGINT SIGTERM
@@ -83,7 +86,6 @@ start_and_exit() {
 }
 
 start() {
-	trap teardown EXIT
 	start_and_exit
 	watch_for_failure
 }
@@ -94,7 +96,6 @@ configure_multisite() {
 }
 
 restart(){
-	trap teardown EXIT
 	sh -c "echo 'restarting' > ${CONTAINER_ARTIFACT_DIR}/splunk-container.state"
 	prep_ansible
 	${SPLUNK_HOME}/bin/splunk stop 2>/dev/null || true

tests/executor.py

@@ -147,7 +147,7 @@ def wait_for_containers(self, count, label=None, name=None, timeout=500):
             for container in containers:
                 # The healthcheck on our Splunk image is not reliable - resorting to checking logs
                 if container.get("Labels", {}).get("maintainer") == "support@splunk.com":
-                    output = self.client.logs(container["Id"], tail=10)
+                    output = self.client.logs(container["Id"], tail=5)
                     if "unable to" in output or "denied" in output or "splunkd.pid file is unreadable" in output:
                         self.logger.error("Container {} did not start properly, last log line: {}".format(container["Names"][0], output))
                     elif "Ansible playbook complete" in output:

tests/test_single_splunk_image.py

@@ -852,7 +852,6 @@ def test_adhoc_1so_run_as_root(self):
         finally:
             if cid:
                 self.client.remove_container(cid, v=True, force=True)
-     
 
     def test_adhoc_1so_declarative_password(self):
         """
@@ -1014,6 +1013,9 @@ def test_adhoc_1so_hec_idempotence(self):
             self.client.exec_start(exec_command)
             # Restart the container - it should pick up the new HEC settings in /tmp/defaults/default.yml
             self.client.restart(splunk_container_name)
+            # After restart, a container's logs are preserved. So, sleep in order for the self.wait_for_containers()
+            # to avoid seeing the prior entrypoint's "Ansible playbook complete" string 
+            time.sleep(15)
             assert self.wait_for_containers(1, name=splunk_container_name)
             assert self.check_splunkd("admin", self.password, name=splunk_container_name)
             # Check the new HEC settings
@@ -1047,6 +1049,9 @@ def test_adhoc_1so_hec_idempotence(self):
             self.client.exec_start(exec_command)
             # Restart the container - it should pick up the new HEC settings in /tmp/defaults/default.yml
             self.client.restart(splunk_container_name)
+            # After restart, a container's logs are preserved. So, sleep in order for the self.wait_for_containers()
+            # to avoid seeing the prior entrypoint's "Ansible playbook complete" string 
+            time.sleep(15)
             assert self.wait_for_containers(1, name=splunk_container_name)
             assert self.check_splunkd("admin", self.password, name=splunk_container_name)
             # Check the new HEC settings
@@ -1078,6 +1083,9 @@ def test_adhoc_1so_hec_idempotence(self):
             self.client.exec_start(exec_command)
             # Restart the container - it should pick up the new HEC settings in /tmp/defaults/default.yml
             self.client.restart(splunk_container_name)
+            # After restart, a container's logs are preserved. So, sleep in order for the self.wait_for_containers()
+            # to avoid seeing the prior entrypoint's "Ansible playbook complete" string 
+            time.sleep(15)
             assert self.wait_for_containers(1, name=splunk_container_name)
             assert self.check_splunkd("admin", self.password, name=splunk_container_name)
             # Check the new HEC settings
@@ -1102,6 +1110,9 @@ def test_adhoc_1so_hec_idempotence(self):
             self.client.exec_start(exec_command)
             # Restart the container - it should pick up the new HEC settings in /tmp/defaults/default.yml
             self.client.restart(splunk_container_name)
+            # After restart, a container's logs are preserved. So, sleep in order for the self.wait_for_containers()
+            # to avoid seeing the prior entrypoint's "Ansible playbook complete" string 
+            time.sleep(15)
             assert self.wait_for_containers(1, name=splunk_container_name)
             assert self.check_splunkd("admin", self.password, name=splunk_container_name)
             # Check the new HEC settings
@@ -1288,13 +1299,15 @@ def test_compose_1so_java_oracle(self):
         assert "java version \"1.8.0" in std_out
         # Restart the container and make sure java is still installed
         self.client.restart("{}_so1_1".format(self.project_name))
+        # After restart, a container's logs are preserved. So, sleep in order for the self.wait_for_containers()
+        # to avoid seeing the prior entrypoint's "Ansible playbook complete" string 
+        time.sleep(15)
         assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name))
         assert self.check_splunkd("admin", self.password)
         exec_command = self.client.exec_create("{}_so1_1".format(self.project_name), "java -version")
         std_out = self.client.exec_start(exec_command)
         assert "java version \"1.8.0" in std_out
 
-
     def test_compose_1so_java_openjdk8(self):
         # Standup deployment
         self.compose_file_name = "1so_java_openjdk8.yaml"
@@ -1322,6 +1335,9 @@ def test_compose_1so_java_openjdk8(self):
         assert "openjdk version \"1.8.0" in std_out
         # Restart the container and make sure java is still installed
         self.client.restart("{}_so1_1".format(self.project_name))
+        # After restart, a container's logs are preserved. So, sleep in order for the self.wait_for_containers()
+        # to avoid seeing the prior entrypoint's "Ansible playbook complete" string 
+        time.sleep(15)
         assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name))
         assert self.check_splunkd("admin", self.password)
         exec_command = self.client.exec_create("{}_so1_1".format(self.project_name), "java -version")
@@ -1356,6 +1372,9 @@ def test_compose_1so_java_openjdk11(self):
         assert "openjdk version \"11.0.2" in std_out
         # Restart the container and make sure java is still installed
         self.client.restart("{}_so1_1".format(self.project_name))
+        # After restart, a container's logs are preserved. So, sleep in order for the self.wait_for_containers()
+        # to avoid seeing the prior entrypoint's "Ansible playbook complete" string 
+        time.sleep(15)
         assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name))
         assert self.check_splunkd("admin", self.password)
         exec_command = self.client.exec_create("{}_so1_1".format(self.project_name), "java -version")

uf/common-files/entrypoint.sh

@@ -28,7 +28,10 @@ setup() {
 
 teardown() {
 	# Always run the stop command on termination
-	${SPLUNK_HOME}/bin/splunk stop 2>/dev/null || true
+	if [ `whoami` != "${SPLUNK_USER}" ]; then
+		RUN_AS_SPLUNK="sudo -u ${SPLUNK_USER}"
+	fi
+	${RUN_AS_SPLUNK} ${SPLUNK_HOME}/bin/splunk stop || true
 }
 
 trap teardown SIGINT SIGTERM
@@ -83,13 +86,11 @@ start_and_exit() {
 }
 
 start() {
-	trap teardown EXIT
 	start_and_exit
 	watch_for_failure
 }
 
 restart(){
-	trap teardown EXIT
 	sh -c "echo 'restarting' > ${CONTAINER_ARTIFACT_DIR}/splunk-container.state"
 	prep_ansible
 	${SPLUNK_HOME}/bin/splunk stop 2>/dev/null || true