Give Splunk some more time after a restart (#668)

We still observe the CI failing because our test suite restarts Splunk.
This change adds a workaround a sleep to let Splunk settle after a restart.

Author: mateusz834

tests/integration/test_app.py

@@ -40,7 +40,7 @@ def setUp(self):
         else:
             logging.debug(f"App {self.app_name} already exists. Skipping creation.")
         if self.service.restart_required:
-            self.service.restart(120)
+            self.restart_splunk()
 
     def tearDown(self):
         super().tearDown()

tests/integration/test_service.py

@@ -27,7 +27,7 @@
 class ServiceTestCase(testlib.SDKTestCase):
     def test_autologin(self):
         service = client.connect(autologin=True, **self.opts.kwargs)
-        self.service.restart(timeout=120)
+        self.restart_splunk()
         reader = service.jobs.oneshot("search index=internal | head 1")
         self.assertIsNotNone(reader)
 
@@ -128,7 +128,7 @@ def test_parse_fail(self):
 
     def test_restart(self):
         service = client.connect(**self.opts.kwargs)
-        self.service.restart(timeout=300)
+        self.restart_splunk()
         service.login()  # Make sure we are awake
 
     def test_read_outputs_with_type(self):
@@ -139,11 +139,11 @@ def test_read_outputs_with_type(self):
         self.assertTrue("tcp", entity.content.type)
 
         if service.restart_required:
-            self.restartSplunk()
+            self.restart_splunk()
         service = client.connect(**self.opts.kwargs)
         client.Entity(service, "data/outputs/tcp/syslog/" + name).delete()
         if service.restart_required:
-            self.restartSplunk()
+            self.restart_splunk()
 
     def test_splunk_version(self):
         service = client.connect(**self.opts.kwargs)
@@ -259,7 +259,7 @@ def test_autologin_with_cookie(self):
             **self.opts.kwargs,
         )
         self.assertTrue(service.has_cookies())
-        self.service.restart(timeout=120)
+        testlib.restart_splunk(self.service)
         reader = service.jobs.oneshot("search index=internal | head 1")
         self.assertIsNotNone(reader)
 
@@ -351,7 +351,8 @@ def test_update_settings(self):
         settings.refresh()
         updated = settings["sessionTimeout"]
         self.assertEqual(updated, original)
-        self.restartSplunk()
+        if self.service.restart_required:
+            self.restart_splunk()
 
 
 class TestTrailing(unittest.TestCase):

tests/testlib.py

@@ -76,6 +76,14 @@ def wait(predicate, timeout=60, pause_time=0.5):
         logging.debug("wait finished after %s seconds", datetime.now() - start)
 
 
+def restart_splunk(service: client.Service):
+    service.restart(timeout=120)
+    # Give Splunk some additional time. In our test suite, a subsequent
+    # restart shortly after the initial restart can cause Splunk to crash
+    # and fail to start.
+    sleep(15)
+
+
 class SDKTestCase(unittest.TestCase):
     restart_already_required = False
     installedApps = []
@@ -138,6 +146,9 @@ def check_entity(self, entity):
                         continue
                 raise
 
+    def restart_splunk(self):
+        restart_splunk(self.service)
+
     def clear_restart_message(self):
         """Tell Splunk to forget that it needs to be restarted.
 
@@ -175,7 +186,7 @@ def install_app_from_collection(self, name):
             if he.status == 400:
                 raise IOError(f"App {name} not found in app collection")
         if self.service.restart_required:
-            self.service.restart(120)
+            self.restart_splunk()
         self.installedApps.append(name)
 
     def app_collection_installed(self):
@@ -221,20 +232,14 @@ def pathInApp(self, appName, pathComponents):
         appPath = separator.join([splunkHome, "etc", "apps", appName] + pathComponents)
         return appPath
 
-    def restartSplunk(self, timeout=240):
-        if self.service.restart_required:
-            self.service.restart(timeout)
-        else:
-            raise NoRestartRequiredError()
-
     @classmethod
     def setUpClass(cls):
         cls.opts = parse([], {}, ".env")
         cls.opts.kwargs.update({"retries": 3})
         # Before we start, make sure splunk doesn't need a restart.
         service = client.connect(**cls.opts.kwargs)
         if service.restart_required:
-            service.restart(timeout=120)
+            self.restart_splunk()
 
     def setUp(self):
         unittest.TestCase.setUp(self)
@@ -244,7 +249,7 @@ def setUp(self):
         # and restart. That way we'll be sane for the rest of
         # the test.
         if self.service.restart_required:
-            self.restartSplunk()
+            self.restart_splunk()
         logging.debug(
             "Connected to splunkd version %s",
             ".".join(str(x) for x in self.service.splunk_version),