add some date preprocessing

Author: frederikhoengaard

python/src/lazylearn/ingestion/ingestion_pipeline_steps/data_parser_step.py

@@ -17,5 +17,6 @@ def apply(self, pipeline: IngestionPipeline):
 
         if isinstance(pipeline.raw_data, DataFrame):
             pipeline.df = pipeline.raw_data
+        # check if raw data is a path to a csv file and read it into csv
         else:
             raise DataSourceError

python/src/lazylearn/ingestion/ingestion_pipeline_steps/interpreter_step.py

@@ -27,6 +27,10 @@ def apply(self, pipeline: IngestionPipeline):
             )  # noqa
 
         pipeline.column_type_map = column_types
+        if "unknown" in pipeline.column_type_map.values():
+            pipeline.needs_type_map = True
+
+        pipeline.type_collections = self.build_type_collections(column_types)
 
     def analyze_column(self, column: Series):
         """
@@ -50,7 +54,7 @@ def analyze_column(self, column: Series):
             column_type = "datetime"
 
         if column_type is None:
-            column_type = "object"
+            column_type = "unknown"
 
         return column_type
 
@@ -110,7 +114,7 @@ def datetime_check(self, column: Series):
             except Exception as e:  # noqa
                 pass
 
-        # if format of values look like dates
+        # if format of values looks like dates
 
         return False
 
@@ -124,3 +128,14 @@ def id_check(self, types, values):
         return all([item == int for item in set(types) if item is not None]) and len(
             set(values)
         ) == len(self.df)
+
+    @staticmethod
+    def build_type_collections(column_type_map):
+        collections = {}
+
+        for data_type in ["datetime", "numeric", "categorical"]:
+            collections[data_type] = [
+                col for col in column_type_map if column_type_map[col] == data_type
+            ]
+
+        return collections

python/src/lazylearn/lazylearn.py

@@ -1,13 +1,28 @@
 from ingestion.ingestion_pipeline import Ingestion
+from preprocessing.time.date_processor import date_processor
 
 
 class LazyLearner:
     def __init__(self):
         self.dataset = None
+        self.task = None
+        self.models = None
+        self.leaderboard = None
 
     def create_project(self, data, target, task="infer"):
         # ingest data
-        self.dataset = Ingestion().run(data)  # noqa
+        self.dataset = Ingestion().run(data)
+
+        if task == "infer":
+            # if target is numeric then regression, else classification
+            if self.dataset.column_type_map[target] == "numeric":
+                self.task = "regression"
+            else:
+                self.task = "classification"
+
+        # process dates
+
+        self.dataset = date_processor(self.dataset)
 
         # preprocess
 

python/src/lazylearn/models/models.py

@@ -2,11 +2,19 @@
 
 
 class Dataset:
-    def __init__(self, df: DataFrame, column_type_map: dict):
+    def __init__(
+        self,
+        df: DataFrame,
+        column_type_map: dict,
+        summary_stats: dict,
+        type_collections: dict,
+    ):
         self.name = None
         self.description = None
         self.df = df
         self.column_type_map = column_type_map
+        self.summary_stats = summary_stats
+        self.type_collections = type_collections
 
     def save(self):
         raise NotImplementedError

python/src/lazylearn/pipeline/pipeline.py

@@ -29,6 +29,23 @@ def __init__(self):
         self.df: DataFrame = None
         self.column_type_map: dict = None
         self.summary_stats: dict = {}
+        self.needs_type_map: bool = False
+        self.type_collections: dict = None
 
     def response(self):
-        return Dataset(df=self.df, column_type_map=self.column_type_map)
+        return Dataset(
+            df=self.df,
+            column_type_map=self.column_type_map,
+            summary_stats=self.summary_stats,
+            type_collections=self.type_collections,
+        )
+
+
+class ModelPipeline(Pipeline):
+    def __init__(self):
+        super().__init__()
+
+
+class RegressionPipeline(ModelPipeline):
+    def __init__(self):
+        super().__init__()

python/src/lazylearn/preprocessing/time/__init__.py

@@ -0,0 +1,34 @@
+from models.models import Dataset
+
+
+def date_processor(dataset: Dataset) -> Dataset:
+    """
+    Method that transform date variables into
+    categorical features.
+
+    :param dataset: Dataset object with date features
+    :return: Dataset object with categorical date
+    features
+    """
+    new_categorical_cols = []
+
+    for date_column in dataset.type_collections["datetime"]:
+        dataset.df[f"{date_column}_year"] = (
+            dataset.df[date_column].dt.isocalendar().year
+        )
+        dataset.df[f"{date_column}_month"] = dataset.df[date_column].dt.month
+        dataset.df[f"{date_column}_week"] = (
+            dataset.df[date_column].dt.isocalendar().week
+        )
+        dataset.df[f"{date_column}_day"] = dataset.df[date_column].dt.isocalendar().day
+
+        new_categorical_cols.append(f"{date_column}_year")
+        new_categorical_cols.append(f"{date_column}_month")
+        new_categorical_cols.append(f"{date_column}_week")
+        new_categorical_cols.append(f"{date_column}_day")
+
+    for cat in new_categorical_cols:
+        dataset.column_type_map[cat] = "categorical"
+        dataset.type_collections["categorical"].append(cat)
+
+    return dataset