Add new global validation using Validation aggregator & errors as dict (to provide more feedback rather than simple msg)

Signed-off-by: David Martin <david@chaosiq.io>

Author: David Martin

chaoslib/activity.py

@@ -16,14 +16,16 @@
     validate_python_activity
 from chaoslib.provider.process import run_process_activity, \
     validate_process_activity
-from chaoslib.types import Activity, Configuration, Experiment, Run, Secrets
+from chaoslib.types import Activity, Configuration, Experiment, Run, Secrets, \
+    ValidationError
+from chaoslib.validation import Validation
 
 
 __all__ = ["ensure_activity_is_valid", "get_all_activities_in_experiment",
            "run_activities"]
 
 
-def ensure_activity_is_valid(activity: Activity):
+def ensure_activity_is_valid(activity: Activity) -> List[ValidationError]:
     """
     Goes through the activity and checks certain of its properties and raise
     :exc:`InvalidActivity` whenever one does not respect the expectations.
@@ -34,75 +36,80 @@ def ensure_activity_is_valid(activity: Activity):
 
     Depending on the type, an activity requires a variety of other keys.
 
-    In all failing cases, raises :exc:`InvalidActivity`.
+    In all failing cases, returns a list of validation errors.
     """
-    errors = []
+    v = Validation()
     if not activity:
-        errors.append(InvalidActivity("empty activity is no activity"))
-        return errors
+        v.add_error("activity", "empty activity is no activity")
+        return v.errors()
 
     # when the activity is just a ref, there is little to validate
     ref = activity.get("ref")
     if ref is not None:
         if not isinstance(ref, str) or ref == '':
-            errors.append(InvalidActivity(
-                "reference to activity must be non-empty strings"))
-        return errors
+            v.add_error(
+                "ref", "reference to activity must be non-empty strings")
+        return v.errors()
 
     activity_type = activity.get("type")
     if not activity_type:
-        errors.append(InvalidActivity("an activity must have a type"))
+        v.add_error("type", "an activity must have a type")
 
     if activity_type not in ("probe", "action"):
-        errors.append(InvalidActivity(
-            "'{t}' is not a supported activity type".format(t=activity_type)))
+        msg = "'{t}' is not a supported activity type".format(t=activity_type)
+        v.add_error("type", msg, value=activity_type)
 
     if not activity.get("name"):
-        errors.append(InvalidActivity("an activity must have a name"))
+        v.add_error("name", "an activity must have a name")
 
     provider = activity.get("provider")
     if not provider:
-        errors.append(InvalidActivity("an activity requires a provider"))
+        v.add_error("provider", "an activity requires a provider")
         provider_type = None
     else:
         provider_type = provider.get("type")
         if not provider_type:
-            errors.append(InvalidActivity("a provider must have a type"))
+            v.add_error("type", "a provider must have a type")
 
         if provider_type not in ("python", "process", "http"):
-            errors.append(InvalidActivity(
-                "unknown provider type '{type}'".format(type=provider_type)))
+            msg = "unknown provider type '{type}'".format(type=provider_type)
+            v.add_error("type", msg, value=provider_type)
 
     timeout = activity.get("timeout")
     if timeout is not None:
         if not isinstance(timeout, numbers.Number):
-            errors.append(
-                InvalidActivity("activity timeout must be a number"))
+            v.add_error(
+                "timeout", "activity timeout must be a number", value=timeout)
 
     pauses = activity.get("pauses")
     if pauses is not None:
         before = pauses.get("before")
         if before is not None and not isinstance(before, numbers.Number):
-            errors.append(
-                InvalidActivity("activity before pause must be a number"))
+            v.add_error(
+                "before", "activity before pause must be a number",
+                value=before
+            )
         after = pauses.get("after")
         if after is not None and not isinstance(after, numbers.Number):
-            errors.append(
-                InvalidActivity("activity after pause must be a number"))
+            v.add_error(
+                "after", "activity after pause must be a number",
+                value=after
+            )
 
     if "background" in activity:
         if not isinstance(activity["background"], bool):
-            errors.append(
-                InvalidActivity("activity background must be a boolean"))
+            v.add_error(
+                "background", "activity background must be a boolean",
+                value=activity["background"])
 
     if provider_type == "python":
-        errors.extend(validate_python_activity(activity))
+        v.extend_errors(validate_python_activity(activity))
     elif provider_type == "process":
-        errors.extend(validate_process_activity(activity))
+        v.extend_errors(validate_process_activity(activity))
     elif provider_type == "http":
-        errors.extend(validate_http_activity(activity))
+        v.extend_errors(validate_http_activity(activity))
 
-    return errors
+    return v.errors()
 
 
 def run_activities(experiment: Experiment, configuration: Configuration,

chaoslib/control/__init__.py

@@ -12,7 +12,8 @@
 from chaoslib.settings import get_loaded_settings
 from chaoslib.types import Settings
 from chaoslib.types import Activity, Configuration, Control as ControlType, \
-    Experiment, Hypothesis, Journal, Run, Secrets
+    Experiment, Hypothesis, Journal, Run, Secrets, ValidationError
+from chaoslib.validation import Validation
 
 
 __all__ = ["controls", "initialize_controls", "cleanup_controls",
@@ -86,11 +87,12 @@ def cleanup_controls(experiment: Experiment):
             cleanup_control(control)
 
 
-def validate_controls(experiment: Experiment) -> List[ChaosException]:
+def validate_controls(experiment: Experiment) -> List[ValidationError]:
     """
     Validate that all declared controls respect the specification.
     """
-    errors = []
+    v = Validation()
+
     controls = get_controls(experiment)
     references = [
         c["name"] for c in get_controls(experiment)
@@ -99,29 +101,33 @@ def validate_controls(experiment: Experiment) -> List[ChaosException]:
     for c in controls:
         if "ref" in c:
             if c["ref"] not in references:
-                errors.append(InvalidControl(
-                    "Control reference '{}' declaration cannot be found"))
+                msg = "Control reference '{}' declaration cannot be found".\
+                    format(c["ref"])
+                v.add_error("ref", msg, value=c["ref"])
 
         if "name" not in c:
-            errors.append(
-                InvalidControl("A control must have a `name` property"))
+            v.add_error("name", "A control must have a `name` property")
 
         name = c.get("name", '')
         if "provider" not in c:
-            errors.append(InvalidControl(
-                "Control '{}' must have a `provider` property".format(name)))
+            v.add_error(
+                "provider",
+                "Control '{}' must have a `provider` property".format(name))
 
         scope = c.get("scope")
         if scope and scope not in ("before", "after"):
-            errors.append(InvalidControl(
+            v.add_error(
+                "scope",
                 "Control '{}' scope property must be 'before' or "
-                "'after' only".format(name)))
+                "'after' only".format(name),
+                value=scope
+            )
 
         provider_type = c.get("provider", {}).get("type")
         if provider_type == "python":
-            errors.extend(validate_python_control(c))
+            v.extend_errors(validate_python_control(c))
 
-    return errors
+    return v.errors()
 
 
 def initialize_global_controls(experiment: Experiment,

chaoslib/control/python.py

@@ -9,7 +9,8 @@
 from chaoslib import substitute
 from chaoslib.exceptions import InvalidActivity, ChaosException
 from chaoslib.types import Activity, Configuration, Control, Experiment, \
-    Journal, Run, Secrets, Settings
+    Journal, Run, Secrets, Settings, ValidationError
+from chaoslib.validation import Validation
 
 
 __all__ = ["apply_python_control", "cleanup_control", "initialize_control",
@@ -83,19 +84,22 @@ def cleanup_control(control: Control):
     func()
 
 
-def validate_python_control(control: Control) -> List[ChaosException]:
+def validate_python_control(control: Control) -> List[ValidationError]:
     """
     Verify that a control block matches the specification
     """
-    errors = []
+    v = Validation()
+
     name = control["name"]
     provider = control["provider"]
     mod_name = provider.get("module")
     if not mod_name:
-        errors.append(InvalidActivity(
-            "Control '{}' must have a module path".format(name)))
+        v.add_error(
+            "module",
+            "Control '{}' must have a module path".format(name)
+        )
         # can not continue any longer - must exit this function
-        return errors
+        return v.errors()
 
     try:
         importlib.import_module(mod_name)

chaoslib/exceptions.py

@@ -47,27 +47,4 @@ class InvalidControl(ChaosException):
 
 
 class ValidationError(ChaosException):
-    def __init__(self, msg, errors, *args, **kwargs):
-        """
-        :param msg: exception message
-        :param errors: single error as string or list of errors/exceptions
-        """
-        if isinstance(errors, str):
-            errors = [errors]
-        self.errors = errors
-        super().__init__(msg, *args, **kwargs)
-
-    def __str__(self) -> str:
-        errors = self.errors
-        nb_errors = len(errors)
-        err_msg = super().__str__()
-        return (
-            "{msg}{dot} {nb} validation error{plural}:\n"
-            " - {errors}".format(
-                msg=err_msg,
-                dot="" if err_msg.endswith(".") else ".",
-                nb=nb_errors,
-                plural="" if nb_errors == 1 else "s",
-                errors="\n - ".join([str(err) for err in errors])
-            )
-        )
+    pass

chaoslib/experiment.py

@@ -15,7 +15,8 @@
     cleanup_global_controls
 from chaoslib.deprecation import warn_about_deprecated_features
 from chaoslib.exceptions import ActivityFailed, ChaosException, \
-    InterruptExecution, InvalidActivity, InvalidExperiment, ValidationError
+    InterruptExecution, InvalidActivity, InvalidExperiment, \
+    ValidationError as ValidationErrorException
 from chaoslib.extension import validate_extensions
 from chaoslib.configuration import load_configuration
 from chaoslib.hypothesis import ensure_hypothesis_is_valid, \
@@ -25,14 +26,17 @@
 from chaoslib.secret import load_secrets
 from chaoslib.settings import get_loaded_settings
 from chaoslib.types import Configuration, Experiment, Journal, Run, Secrets, \
-    Settings
+    Settings, ValidationError
+from chaoslib.validation import Validation
 
 initialize_global_controls
 __all__ = ["ensure_experiment_is_valid", "run_experiment", "load_experiment"]
 
 
 @with_cache
-def ensure_experiment_is_valid(experiment: Experiment):
+def ensure_experiment_is_valid(experiment: Experiment,
+                               no_raise: bool = False
+                               ) -> List[ValidationError]:
     """
     A chaos experiment consists of a method made of activities to carry
     sequentially.
@@ -52,66 +56,78 @@ def ensure_experiment_is_valid(experiment: Experiment):
     if the experiment is not valid.
     If multiple validation errors are found, the errors are listed
     as part of the exception message
+
+    If `no_raise` is True, the function will not raise any
+    exception but rather return the list of validation errors
     """
     logger.info("Validating the experiment's syntax")
 
-    full_validation_msg = 'Experiment is not valid, ' \
-                          'please fix the following errors'
-    errors = []
+    full_validation_msg = "Experiment is not valid, " \
+                          "please fix the following errors. " \
+                          "\n{}"
+    v = Validation()
 
     if not experiment:
+        v.add_error("$", "an empty experiment is not an experiment")
         # empty experiment, cannot continue validation any further
-        raise ValidationError(full_validation_msg,
-                              "an empty experiment is not an experiment")
+        if no_raise:
+            return v.errors()
+        raise InvalidExperiment(full_validation_msg.format(str(v)))
 
     if not experiment.get("title"):
-        errors.append(InvalidExperiment("experiment requires a title"))
+        v.add_error("title", "experiment requires a title")
 
     if not experiment.get("description"):
-        errors.append(InvalidExperiment("experiment requires a description"))
+        v.add_error("description", "experiment requires a description")
 
     tags = experiment.get("tags")
     if tags:
         if list(filter(lambda t: t == '' or not isinstance(t, str), tags)):
-            errors.append(InvalidExperiment(
-                "experiment tags must be a non-empty string"))
+            v.add_error("tags", "experiment tags must be a non-empty string")
 
-    errors.extend(validate_extensions(experiment))
+    v.extend_errors(validate_extensions(experiment))
 
     config = load_configuration(experiment.get("configuration", {}))
     load_secrets(experiment.get("secrets", {}), config)
 
-    errors.extend(ensure_hypothesis_is_valid(experiment))
+    v.extend_errors(ensure_hypothesis_is_valid(experiment))
 
     method = experiment.get("method")
     if not method:
-        errors.append(InvalidExperiment("an experiment requires a method with "
-                                        "at least one activity"))
+        v.add_error(
+            "method",
+            "an experiment requires a method with at least one activity")
     else:
         for activity in method:
-            errors.extend(ensure_activity_is_valid(activity))
+            v.extend_errors(ensure_activity_is_valid(activity))
 
             # let's see if a ref is indeed found in the experiment
             ref = activity.get("ref")
             if ref and not lookup_activity(ref):
-                errors.append(
-                    InvalidActivity("referenced activity '{r}' could not be "
-                                    "found in the experiment".format(r=ref)))
+                v.add_error(
+                    "ref",
+                    "referenced activity '{r}' could not be "
+                    "found in the experiment".format(r=ref),
+                    value=ref
+                )
 
     rollbacks = experiment.get("rollbacks", [])
     for activity in rollbacks:
-        errors.extend(ensure_activity_is_valid(activity))
+        v.extend_errors(ensure_activity_is_valid(activity))
 
     warn_about_deprecated_features(experiment)
 
-    errors.extend(validate_controls(experiment))
+    v.extend_errors(validate_controls(experiment))
 
-    if errors:
-        raise ValidationError(full_validation_msg, errors)
+    if v.has_errors():
+        if no_raise:
+            return v.errors()
+        raise InvalidExperiment(full_validation_msg.format(str(v)))
 
     logger.info("Experiment looks valid")
 
 
+
 def initialize_run_journal(experiment: Experiment) -> Journal:
     return {
         "chaoslib-version": __version__,