Merge pull request #179 from chaostoolkit/chaostoolkit/issue176

Add rollback runtime strategies

Author: Lawouach

CHANGELOG.md

@@ -4,6 +4,25 @@
 
 [Unreleased]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/1.10.0...HEAD
 
+### Added
+
+- Added runtime strategies for rollback as per [chaostoolkit#176][].
+  Until now, they were never played
+  an activity would fail during the hypothesis or if the execution
+  was interrupted from a control. With the strategies, you can now decide
+  that they are always applied, never or only when the experiment deviated.
+  This is a flag passed to the settings as follows:
+  
+  ```
+  runtime:
+    rollbacks:
+      strategy: "always|never|default|deviated"
+  ```
+
+  The `"default"` strategy remains backward compatible.
+
+  [chaostoolkit#176]: https://github.com/chaostoolkit/chaostoolkit/issues/176
+
 ## [1.10.0][] - 2020-06-19
 
 [1.10.0]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/1.9.0...1.10.0

chaoslib/experiment.py

@@ -190,6 +190,8 @@ def run_experiment(experiment: Experiment,
     initialize_global_controls(experiment, config, secrets, settings)
     initialize_controls(experiment, config, secrets)
     activity_pool, rollback_pool = get_background_pools(experiment)
+    rollback_strategy = settings.get("runtime", {}).get(
+        "rollbacks", {}).get("strategy", "default")
 
     experiment["title"] = substitute(experiment["title"], config, secrets)
     logger.info("Running experiment: {t}".format(t=experiment["title"]))
@@ -254,6 +256,34 @@ def run_experiment(experiment: Experiment,
                            "leaving without applying rollbacks.")
         else:
             journal["status"] = journal["status"] or "completed"
+
+        has_deviated = journal["deviated"]
+        journal_status = journal["status"]
+        play_rollbacks = False
+        if rollback_strategy == "always":
+            logger.warning(
+                "Rollbacks were explicitly requested to be played")
+            play_rollbacks = True
+        elif rollback_strategy == "never":
+            logger.warning(
+                "Rollbacks were explicitly requested to not be played")
+            play_rollbacks = False
+        elif rollback_strategy == "default" and \
+                journal_status not in ["failed", "interrupted"]:
+            play_rollbacks = True
+        elif rollback_strategy == "deviated":
+            if has_deviated:
+                logger.warning(
+                    "Rollbacks will be played only because the experiment "
+                    "deviated")
+                play_rollbacks = True
+            else:
+                logger.warning(
+                    "Rollbacks werre explicitely requested to be played only "
+                    "if the experiment deviated. Since this is not the case, "
+                    "we will not play them.")
+
+        if play_rollbacks:
             try:
                 journal["rollbacks"] = apply_rollbacks(
                     experiment, config, secrets, rollback_pool, dry)
@@ -269,8 +299,7 @@ def run_experiment(experiment: Experiment,
         journal["end"] = datetime.utcnow().isoformat()
         journal["duration"] = time.time() - started_at
 
-        has_deviated = journal["deviated"]
-        status = "deviated" if has_deviated else journal["status"]
+        status = "deviated" if has_deviated else journal_status
 
         logger.info(
             "Experiment ended with status: {s}".format(s=status))

tests/fixtures/actions.py

@@ -1,5 +1,48 @@
-# -*- coding: utf-8 -*-
-import sys
+EmptyAction = {}
 
 
-EmptyAction = {}
+DoNothingAction = {
+    "name": "a name",
+    "type": "action",
+    "provider": {
+        "type": "python",
+        "module": "fixtures.fakeext",
+        "func": "do_nothing"
+    }
+}
+
+
+EchoAction = {
+    "name": "a name",
+    "type": "action",
+    "provider": {
+        "type": "python",
+        "module": "fixtures.fakeext",
+        "func": "echo_message",
+        "arguments": {
+            "message": "kaboom"
+        }
+    }
+}
+
+
+FailAction = {
+    "name": "a name",
+    "type": "action",
+    "provider": {
+        "type": "python",
+        "module": "fixtures.fakeext",
+        "func": "force_failed_activity"
+    }
+}
+
+
+InterruptAction = {
+    "name": "a name",
+    "type": "action",
+    "provider": {
+        "type": "python",
+        "module": "fixtures.fakeext",
+        "func": "force_interrupting_experiment"
+    }
+}

tests/fixtures/experiments.py

@@ -2,6 +2,8 @@
 from copy import deepcopy
 import os
 
+from fixtures.actions import DoNothingAction, EchoAction, FailAction, \
+    InterruptAction
 from fixtures.probes import BackgroundPythonModuleProbe, MissingFuncArgProbe, \
     PythonModuleProbe, PythonModuleProbeWithBoolTolerance, \
     PythonModuleProbeWithExternalTolerance, PythonModuleProbeWithLongPause, \
@@ -11,7 +13,7 @@
     PythonModuleProbeWithProcessStatusTolerance, \
     PythonModuleProbeWithProcessFailedStatusTolerance, \
     PythonModuleProbeWithProcesStdoutTolerance, \
-    PythonModuleProbeWithHTTPStatusToleranceDeviation
+    PythonModuleProbeWithHTTPStatusToleranceDeviation, FailProbe
 
 Secrets = {}
 
@@ -421,3 +423,75 @@
       path: {}
     timeout: 30
 """.format(os.path.abspath(__file__))
+
+
+ExperimentWithRegularRollback = {
+    "title": "do cats live in the Internet?",
+    "description": "an experiment of importance",
+    "steady-state-hypothesis": {
+        "title": "hello"
+    },
+    "method": [
+        EchoAction
+    ],
+    "rollbacks": [
+        EchoAction
+    ]
+}
+
+
+ExperimentWithFailedActionInMethodAndARollback = {
+    "title": "do cats live in the Internet?",
+    "description": "an experiment of importance",
+    "steady-state-hypothesis": {
+        "title": "hello"
+    },
+    "method": [
+        FailAction
+    ],
+    "rollbacks": [
+        EchoAction
+    ]
+}
+
+
+ExperimentWithFailedActionInSSHAndARollback = {
+    "title": "do cats live in the Internet?",
+    "description": "an experiment of importance",
+    "steady-state-hypothesis": {
+        "title": "hello",
+        "probes": [
+            FailProbe
+        ]
+    },
+    "method": [
+        DoNothingAction
+    ],
+    "rollbacks": [
+        EchoAction
+    ]
+}
+
+
+ExperimentWithInterruptedExperimentAndARollback = {
+    "title": "do cats live in the Internet?",
+    "description": "an experiment of importance",
+    "steady-state-hypothesis": {
+        "title": "hello"
+    },
+    "method": [
+        deepcopy(EchoAction)
+    ],
+    "rollbacks": [
+        EchoAction
+    ]
+}
+ExperimentWithInterruptedExperimentAndARollback["method"][0]["controls"] = [
+    {
+        "name": "dummy",
+        "provider": {
+            "type": "python",
+            "module": "fixtures.interruptexperiment"
+        }
+    }
+]

tests/fixtures/fakeext.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 from typing import List
 
-from chaoslib.exceptions import InterruptExecution
+from chaoslib.exceptions import ActivityFailed, InterruptExecution
 
 __all__ = ["many_args", "many_args_with_rich_types", "no_args_docstring",
            "no_args", "one_arg", "one_untyped_arg", "one_arg_with_default",
@@ -66,3 +66,20 @@ def many_args_with_rich_types(message: str, recipients: List[str],
     Many arguments with rich types.
     """
     pass
+
+
+def do_nothing():
+    pass
+
+
+def echo_message(message: str) -> str:
+    print(message)
+    return message
+
+
+def force_failed_activity():
+    raise ActivityFailed()
+
+
+def force_interrupting_experiment():
+    raise InterruptExecution()

tests/fixtures/interruptexperiment.py

@@ -0,0 +1,5 @@
+from chaoslib.exceptions import InterruptExecution
+
+
+def after_activity_control(**kwargs):
+    raise InterruptExecution()

tests/fixtures/probes.py

@@ -373,3 +373,14 @@ def must_be_in_range(a: int, b: int, value: Any = None) -> bool:
         raise ActivityFailed("body is not in expected range")
     else:
         return True
+
+FailProbe = {
+    "name": "a name",
+    "type": "probe",
+    "tolerance": True,
+    "provider": {
+        "type": "python",
+        "module": "fixtures.fakeext",
+        "func": "force_failed_activity"
+    }
+}