Merge pull request #182 from chaostoolkit/chaostoolkit/issue175-revisited

Add var and var files support

Author: Lawouach

CHANGELOG.md

@@ -4,11 +4,19 @@
 
 [Unreleased]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/1.11.1...HEAD
 
+### Added
+
+- Added ways to override configuration and secrets from var files or passed to
+  the chaostoolkit cli [chaostoolkit#175][].
+
+
+[chaostoolkit#175]: https://github.com/chaostoolkit/chaostoolkit/issues/175
+
 ## [1.11.1][] - 2020-07-29
 
 [1.11.1]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/1.11.0...1.11.1
 
-## Changed
+### Changed
 
 - Controls can now update/change configuration and secrets on the fly as per
   the specification [#181][181]

chaoslib/__init__.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 from collections import ChainMap
+import os.path
 from string import Template
-from typing import Any, Dict, List, Mapping, Union
+from typing import Any, Dict, List, Mapping, Tuple, Union
 
 HAS_CHARDET = True
 try:
@@ -12,11 +13,19 @@
     except ImportError:
         HAS_CHARDET = False
 from logzero import logger
+try:
+    import simplejson as json
+    from simplejson.errors import JSONDecodeError
+except ImportError:
+    import json
+    from json.decoder import JSONDecodeError
+import yaml
 
 from chaoslib.exceptions import ActivityFailed
-from chaoslib.types import Configuration, Secrets
+from chaoslib.types import Configuration, ConfigVars, Secrets, SecretVars
 
-__all__ = ["__version__", "decode_bytes", "substitute"]
+__all__ = ["__version__", "decode_bytes", "substitute", "merge_vars",
+           "convert_vars"]
 __version__ = '1.11.1'
 
 
@@ -119,3 +128,126 @@ def decode_bytes(data: bytes, default_encoding: str = 'utf-8') -> str:
     except UnicodeDecodeError:
         raise ActivityFailed(
             "Failed to decode bytes using encoding '{}'".format(encoding))
+
+
+def merge_vars(var: Dict[str, Union[str, float, int, bytes]] = None,  # noqa: C901
+               var_files: List[str] = None) -> Tuple[ConfigVars, SecretVars]:
+    """
+    Load configuration and secret values from the given set of variables.
+    These values are applicable for substitution when the experiment runs.
+    If `var` is set, it must be a dictionary which will be used as
+    configuration values only.
+    If `var_files` is set, it can be a list of any of these two items:
+    * a Json or Yaml payload which must be also mappings with two top-level
+      keys: `"configuration"` and `"secrets"`. If any is present, it must
+      respect the format of the confiuration and secrets of the experiment
+      format.
+    * a .env file which is used to load environment variable on the fly.
+      In that case, the values are injected in the process environment so that
+      they get picked up during experiment's execution as if they had been
+      from the terminal itself.
+    Note that, when multiple var files are provided, they can override each
+    other.
+    The output of this function is a tuple made of configuration and secrets
+    that will be used during the experiment's execution for lookup.
+    """
+    config_vars = {}
+    secret_vars = {}
+
+    if var_files:
+        for var_file in var_files:
+            logger.debug("Loading var from file '{}'".format(var_file))
+
+            if not os.path.isfile(var_file):
+                logger.error("Cannot read var file '{}'".format(var_file))
+                continue
+
+            content = None
+            with open(var_file) as f:
+                content = f.read()
+
+            if not content:
+                logger.debug("Var file '{}' is empty".format(var_file))
+                continue
+
+            data = None
+            _, ext = os.path.splitext(var_file)
+            if ext in (".yaml", ".yml"):
+                try:
+                    data = yaml.safe_load(content)
+                except yaml.YAMLError as y:
+                    logger.error(
+                        "Failed to parse variable file '{}': {}".format(
+                            var_file, str(y)))
+                    continue
+            elif ext in (".json"):
+                try:
+                    data = json.loads(content)
+                except JSONDecodeError as x:
+                    logger.error(
+                        "Failed to parse variable file '{}': {}".format(
+                            var_file, str(x)))
+                    continue
+
+            # process .env files
+            if not data:
+                for line in content.split(os.linesep):
+                    line = line.strip()
+                    if not line or line.startswith("#"):
+                        continue
+
+                    k, v = line.split('=', 1)
+                    os.environ[k] = v
+                    logger.debug(
+                        "Inject environment variable '{}' from "
+                        "file '{}'".format(k, var_file))
+            else:
+                logger.debug(
+                    "Reading configuration/secrets from {}".format(f.name))
+                config_vars.update(data.get("configuration", {}))
+                secret_vars.update(data.get("secrets", {}))
+
+    if var:
+        for k in var:
+            logger.debug("Using configuration variable '{}'".format(k))
+            config_vars[k] = var[k]
+
+    return (config_vars, secret_vars)
+
+
+def convert_vars(value: List[str]) -> Dict[str, Any]:  # noqa: C901
+    """
+    Process all variables and return a dictionnary of them with the
+    value converted to the appropriate type.
+
+    The list of values is as follows: `key[:type]=value` with `type` being one
+    of: str, int, float and bytes. `str` is the default and can be omitted.
+    """
+    var = {}
+    for v in value:
+        try:
+            k, v = v.split('=', 1)
+            if ':' in k:
+                k, typ = k.rsplit(':', 1)
+                try:
+                    if typ == 'str':
+                        pass
+                    elif typ == 'int':
+                        v = int(v)
+                    elif typ == 'float':
+                        v = float(v)
+                    elif typ == 'bytes':
+                        v = v.encode('utf-8')
+                    else:
+                        raise ValueError(
+                            'var supports only: str, int, float and bytes')
+                except (TypeError, UnicodeEncodeError):
+                    raise ValueError(
+                        'var cannot convert value to required type')
+            var[k] = v
+        except ValueError:
+            raise
+        except Exception:
+            raise ValueError('var needs to be in the format name[:type]=value')
+
+    return var

chaoslib/caching.py

@@ -3,7 +3,7 @@
 # referenced from other places in the experiment
 from functools import wraps
 import inspect
-from typing import List, Union
+from typing import Any, List, Dict, Union
 
 from logzero import logger
 
@@ -49,7 +49,8 @@ def with_cache(f):
     function.
     """
     @wraps(f)
-    def wrapped(experiment: Experiment, settings: Settings = None):
+    def wrapped(experiment: Experiment, settings: Settings = None,
+                experiment_vars: Dict[str, Any] = None):
         try:
             if experiment:
                 cache_activities(experiment)
@@ -58,8 +59,13 @@ def wrapped(experiment: Experiment, settings: Settings = None):
             arguments = {
                 "experiment": experiment
             }
+
             if "settings" in sig.parameters:
                 arguments["settings"] = settings
+
+            if "experiment_vars" in sig.parameters:
+                arguments["experiment_vars"] = experiment_vars
+
             return f(**arguments)
         finally:
             clear_cache()

chaoslib/configuration.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import os
-from typing import Dict
+from typing import Any, Dict
 
 from logzero import logger
 
@@ -10,7 +10,8 @@
 __all__ = ["load_configuration"]
 
 
-def load_configuration(config_info: Dict[str, str]) -> Configuration:
+def load_configuration(config_info: Dict[str, str],
+                       extra_vars: Dict[str, Any] = None) -> Configuration:
     """
     Load the configuration. The `config_info` parameter is a mapping from
     key strings to value as strings or dictionaries. In the former case, the
@@ -43,22 +44,30 @@ def load_configuration(config_info: Dict[str, str]) -> Configuration:
     variable. The `host` configuration key is dynamically fetched from the
     `HOSTNAME` environment variable, but if not defined, the default value
     `localhost` will be used instead.
+
+    When `extra_vars` is provided, it must be a dictionnary where keys map
+    to configuration key. The values from `extra_vars` always override the
+    values from the experiment itself. This is useful to the Chaos Toolkit
+    CLI mostly to allow overriding values directly from cli arguments. It's
+    seldom required otherwise.
     """
     logger.debug("Loading configuration...")
     env = os.environ
+    extra_vars = extra_vars or {}
     conf = {}
 
     for (key, value) in config_info.items():
         if isinstance(value, dict) and "type" in value:
             if value["type"] == "env":
                 env_key = value["key"]
                 env_default = value.get("default")
-                if (env_key not in env) and (env_default is None):
+                if (env_key not in env) and (env_default is None) and \
+                        (key not in extra_vars):
                     raise InvalidExperiment(
                         "Configuration makes reference to an environment key"
                         " that does not exist: {}".format(env_key))
-                conf[key] = env.get(env_key, env_default)
+                conf[key] = extra_vars.get(key, env.get(env_key, env_default))
         else:
-            conf[key] = value
+            conf[key] = extra_vars.get(key, value)
 
     return conf

chaoslib/experiment.py

@@ -3,7 +3,7 @@
 from datetime import datetime
 import platform
 import time
-from typing import List
+from typing import Any, Dict, List
 
 from logzero import logger
 
@@ -157,8 +157,8 @@ def get_background_pools(experiment: Experiment) -> ThreadPoolExecutor:
 
 
 @with_cache
-def run_experiment(experiment: Experiment,
-                   settings: Settings = None) -> Journal:
+def run_experiment(experiment: Experiment, settings: Settings = None,
+                   experiment_vars: Dict[str, Any] = None) -> Journal:
     """
     Run the given `experiment` method step by step, in the following sequence:
     steady probe, action, close probe.
@@ -185,7 +185,9 @@ def run_experiment(experiment: Experiment,
 
     started_at = time.time()
     settings = settings if settings is not None else get_loaded_settings()
-    config = load_configuration(experiment.get("configuration", {}))
+    config_vars, secret_vars = experiment_vars or (None, None)
+    config = load_configuration(
+        experiment.get("configuration", {}), extra_vars=config_vars)
     secrets = load_secrets(experiment.get("secrets", {}), config)
     initialize_global_controls(experiment, config, secrets, settings)
     initialize_controls(experiment, config, secrets)

chaoslib/types.py

@@ -5,7 +5,7 @@
            "TargetLayers", "Activity", "Journal", "Run", "Secrets", "Step",
            "Configuration", "Discovery", "DiscoveredActivities", "Extension",
            "DiscoveredSystemInfo", "Settings", "EventPayload", "Tolerance",
-           "Hypothesis", "Control"]
+           "Hypothesis", "Control", "ConfigVars", "SecretVars"]
 
 
 Action = Dict[str, Any]
@@ -37,3 +37,6 @@
 Extension = Dict[str, Any]
 Hypothesis = Dict[str, Any]
 Control = Dict[str, Any]
+
+ConfigVars = Dict[str, Any]
+SecretVars = Dict[str, Any]