Completes OPEN-3507 Validate column names to not allow special characters

Author: gustavocidornelas

openlayer/__init__.py

@@ -36,7 +36,6 @@ class OpenlayerClient(object):
 
     def __init__(self, api_key: str = None):
         self.api = api.Api(api_key)
-        # self.subscription_plan = self.api.get_request("me/subscription-plan")
 
         if not os.path.exists(OPENLAYER_DIR):
             os.makedirs(OPENLAYER_DIR)

openlayer/schemas.py

@@ -6,7 +6,18 @@
 from .models import ModelType
 from .tasks import TaskType
 
-
+# ---------------------------- Regular expressions --------------------------- #
+COLUMN_NAME_REGEX = validate = ma.validate.Regexp(
+    r"^[a-zA-Z0-9_-]+$",
+    error="strings that are not alphanumeric with underscores or hyphens."
+    + " Spaces and special characters are not allowed.",
+)
+LANGUAGE_CODE_REGEX = ma.validate.Regexp(
+    r"^[a-z]{2}(-[A-Z]{2})?$",
+    error="`language` of the dataset is not in the ISO 639-1 (alpha-2 code) format.",
+)
+
+# ---------------------------------- Schemas --------------------------------- #
 class CommitSchema(ma.Schema):
     """Schema for commits."""
 
@@ -23,11 +34,13 @@ class DatasetSchema(ma.Schema):
     """Schema for datasets."""
 
     categoricalFeatureNames = ma.fields.List(
-        ma.fields.Str(), allow_none=True, load_default=[]
+        ma.fields.Str(validate=COLUMN_NAME_REGEX),
+        allow_none=True,
+        load_default=[],
     )
     classNames = ma.fields.List(ma.fields.Str(), required=True)
     columnNames = ma.fields.List(
-        ma.fields.Str(),
+        ma.fields.Str(validate=COLUMN_NAME_REGEX),
         required=True,
     )
     label = ma.fields.Str(
@@ -40,21 +53,26 @@ class DatasetSchema(ma.Schema):
         required=True,
     )
     featureNames = ma.fields.List(
-        ma.fields.Str(),
+        ma.fields.Str(validate=COLUMN_NAME_REGEX),
         load_default=[],
     )
-    labelColumnName = ma.fields.Str(required=True)
+    labelColumnName = ma.fields.Str(
+        validate=COLUMN_NAME_REGEX,
+        required=True,
+    )
     language = ma.fields.Str(
         load_default="en",
-        validate=ma.validate.Regexp(
-            r"^[a-z]{2}(-[A-Z]{2})?$",
-            error="`language` of the dataset is not in the ISO 639-1 (alpha-2 code) format.",
-        ),
+        validate=LANGUAGE_CODE_REGEX,
     )
     metadata = ma.fields.Dict(allow_none=True, load_default={})
-    predictionsColumnName = ma.fields.Str(allow_none=True, load_default=None)
+    predictionsColumnName = ma.fields.Str(
+        validate=COLUMN_NAME_REGEX,
+        allow_none=True,
+        load_default=None,
+    )
     sep = ma.fields.Str(load_default=",")
     textColumnName = ma.fields.Str(
+        validate=COLUMN_NAME_REGEX,
         allow_none=True,
     )
 
@@ -70,7 +88,10 @@ def validates_label_column_not_in_feature_names(self, data, **kwargs):
 class ModelSchema(ma.Schema):
     """Schema for models with artifacts (i.e., model_package)."""
 
-    categoricalFeatureNames = ma.fields.List(ma.fields.Str(), load_default=[])
+    categoricalFeatureNames = ma.fields.List(
+        ma.fields.Str(validate=COLUMN_NAME_REGEX),
+        load_default=[],
+    )
     classNames = ma.fields.List(
         ma.fields.Str(),
     )
@@ -81,7 +102,11 @@ class ModelSchema(ma.Schema):
             max=64,
         ),
     )
-    featureNames = ma.fields.List(ma.fields.Str(), allow_none=True, load_default=[])
+    featureNames = ma.fields.List(
+        ma.fields.Str(validate=COLUMN_NAME_REGEX),
+        allow_none=True,
+        load_default=[],
+    )
     metadata = ma.fields.Dict(
         allow_none=True,
         load_default={},

openlayer/validators.py

@@ -235,7 +235,7 @@ def validate(self) -> List[str]:
         self._validate_bundle_resources()
 
         if not self.failed_validations:
-            print("All validations passed!")
+            print("All commit bundle validations passed!")
 
         return self.failed_validations
 
@@ -287,7 +287,7 @@ def validate(self) -> List[str]:
         self._validate_commit_message()
 
         if not self.failed_validations:
-            print("All validations passed!")
+            print("All commit validations passed!")
 
         return self.failed_validations
 
@@ -454,6 +454,9 @@ def _validate_dataset_and_config_consistency(self):
         if self.dataset_config and self.dataset_df is not None:
             # Extract vars
             dataset_df = self.dataset_df
+            categorical_feature_names = self.dataset_config.get(
+                "categoricalFeatureNames"
+            )
             class_names = self.dataset_config.get("classNames")
             column_names = self.dataset_config.get("columnNames")
             label_column_name = self.dataset_config.get("labelColumnName")
@@ -582,6 +585,14 @@ def _validate_dataset_and_config_consistency(self):
                         "There are features specified in `featureNames` which are "
                         "not in the dataset."
                     )
+                if categorical_feature_names:
+                    if self._columns_not_in_dataset_df(
+                        dataset_df, categorical_feature_names
+                    ):
+                        dataset_and_config_consistency_failed_validations.append(
+                            "There are categorical features specified in `categoricalFeatureNames` "
+                            "which are not in the dataset."
+                        )
 
         # Print results of the validation
         if dataset_and_config_consistency_failed_validations:
@@ -723,7 +734,7 @@ def validate(self) -> List[str]:
         self._validate_dataset_and_config_consistency()
 
         if not self.failed_validations:
-            print("All validations passed!")
+            print("All dataset validations passed!")
 
         return self.failed_validations
 
@@ -1070,7 +1081,7 @@ def validate(self):
         self._validate_project_config()
 
         if not self.failed_validations:
-            print("All validations passed!")
+            print("All model validations passed!")
 
         return self.failed_validations