Catch InterruptExecption during rollbacks

Closes #132

Signed-off-by: Sylvain Hellegouarch <sh@defuze.org>

Author: Lawouach

CHANGELOG.md

@@ -4,6 +4,13 @@
 
 [Unreleased]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/1.7.0...HEAD
 
+### Added
+
+- Catch `InterruptExecution` during rollbacks so that the experiment terminates
+  gracefully [#132][132]
+
+[132]: https://github.com/chaostoolkit/chaostoolkit-lib/issues/132
+
 ## [1.7.0][] - 2019-09-21
 
 [1.7.0]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/1.6.0...1.7.0

chaoslib/experiment.py

@@ -248,8 +248,12 @@ def run_experiment(experiment: Experiment,
                         "leaving without applying rollbacks.")
         else:
             journal["status"] = journal["status"] or "completed"
-            journal["rollbacks"] = apply_rollbacks(
-                experiment, config, secrets, rollback_pool, dry)
+            try:
+                journal["rollbacks"] = apply_rollbacks(
+                    experiment, config, secrets, rollback_pool, dry)
+            except InterruptExecution as i:
+                journal["status"] = "interrupted"
+                logger.fatal(str(i))
 
         journal["end"] = datetime.utcnow().isoformat()
         journal["duration"] = time.time() - started_at

tests/fixtures/experiments.py

@@ -89,6 +89,20 @@
     ]
 }
 
+ExperimentWithRollbackLongPause = {
+    "title": "do cats live in the Internet?",
+    "description": "an experiment of importance",
+    "steady-state-hypothesis": {
+        "title": "hello"
+    },
+    "method": [
+        PythonModuleProbe
+    ],
+    "rollbacks": [
+        PythonModuleProbeWithLongPause
+    ]
+}
+
 ExperimentWithLongPauseBefore = deepcopy(ExperimentWithLongPause)
 ExperimentWithLongPauseBefore["method"][1] = BackgroundPythonModuleProbeWithLongPauseBefore
 

tests/test_experiment.py

@@ -12,7 +12,7 @@
 import yaml
 
 from chaoslib.exceptions import ActivityFailed, InvalidActivity, \
-    InvalidExperiment
+    InvalidExperiment, InterruptExecution
 from chaoslib.experiment import ensure_experiment_is_valid, load_experiment, \
     run_experiment, run_activities
 from chaoslib.types import Experiment
@@ -172,6 +172,21 @@ def handler(signum, frame):
         pytest.fail("we should have swalled the SystemExit exception")
 
 
+def test_can_interrupt_rollbacks():
+    def handler(signum, frame):
+        raise InterruptExecution("boom")
+
+    signal.signal(signal.SIGALRM, handler)
+    signal.alarm(1)
+
+    try:
+        journal = run_experiment(experiments.ExperimentWithRollbackLongPause)
+        assert isinstance(journal, dict)
+        assert journal["status"] == "interrupted"
+    except Exception:
+        pytest.fail("we should have swalled the InterruptExecution exception")
+
+
 def test_probes_can_reference_each_other():
     experiment = experiments.RefProbeExperiment.copy()
     experiment["dry"] = True