durations and simple splits

Author: frederikhoengaard

python/src/lazylearn/ingestion/ingestion_pipeline_steps/data_parser_step.py

@@ -1,4 +1,6 @@
+import pandas as pd
 from errors.errors import DataSourceError
+from ingestion.utils.csv import csv_check
 from pandas import DataFrame
 from pipeline.pipeline import IngestionPipeline, PipelineStep
 
@@ -18,5 +20,7 @@ def apply(self, pipeline: IngestionPipeline):
         if isinstance(pipeline.raw_data, DataFrame):
             pipeline.df = pipeline.raw_data
         # check if raw data is a path to a csv file and read it into csv
+        elif csv_check(pipeline.df):
+            pipeline.df = pd.read_csv(pipeline.raw_data)
         else:
             raise DataSourceError

python/src/lazylearn/ingestion/utils/__init__.py

@@ -0,0 +1,2 @@
+def csv_check(path):
+    raise NotImplementedError

python/src/lazylearn/ingestion/utils/csv.py

@@ -1,13 +1,16 @@
 from ingestion.ingestion_pipeline import Ingestion
+from model_selection.splitters import test_train_splitter
 from preprocessing.time.date_processor import date_processor
+from preprocessing.time.duration import duration_builder
 
 
 class LazyLearner:
-    def __init__(self):
+    def __init__(self, random_state=None):
         self.dataset = None
         self.task = None
         self.models = None
         self.leaderboard = None
+        self.random_state = random_state
 
     def create_project(self, data, target, task="infer"):
         # ingest data
@@ -23,11 +26,19 @@ def create_project(self, data, target, task="infer"):
         # process dates
 
         self.dataset = date_processor(self.dataset)
+        self.dataset = duration_builder(self.dataset)
 
-        # preprocess
+        # split partitions
+
+        self.dataset = test_train_splitter(self.dataset, random_state=self.random_state)
 
         # set modelling configurations
 
+    def run_autopilot(self):
+        raise NotImplementedError
+
+        # preprocess
+
         # train
 
         # eval

python/src/lazylearn/lazylearn.py

@@ -0,0 +1,17 @@
+from models.models import Dataset
+from sklearn.model_selection import train_test_split
+
+
+def test_train_splitter(dataset: Dataset, random_state=None) -> Dataset:
+    train_partition, test_partition = train_test_split(
+        dataset.df, test_size=0.2, random_state=random_state
+    )
+
+    dataset.partitions["test"] = test_partition
+    dataset.partitions["train"] = train_partition
+
+    return dataset
+
+
+def cv_splitter(dataset: Dataset) -> Dataset:
+    return dataset

python/src/lazylearn/model_selection/__init__.py

@@ -15,6 +15,7 @@ def __init__(
         self.column_type_map = column_type_map
         self.summary_stats = summary_stats
         self.type_collections = type_collections
+        self.partitions: dict = {}
 
     def save(self):
         raise NotImplementedError

python/src/lazylearn/model_selection/splitters.py

@@ -0,0 +1,24 @@
+from models.models import Dataset
+
+
+def duration_builder(dataset: Dataset) -> Dataset:
+    """
+
+    :param dataset:
+    :return:
+    """
+    date_cols = dataset.type_collections.get("datetime")
+
+    if len(date_cols) > 1:
+        for i in range(len(date_cols)):
+            for j in range(i + 1, len(date_cols)):
+                col_name = f"duration({date_cols[i]}-{date_cols[j]})"
+                dataset.df[col_name] = (
+                    (dataset.df[date_cols[i]] - dataset.df[date_cols[j]])
+                    .astype("timedelta64[D]")
+                    .astype(int)
+                )
+                dataset.column_type_map[col_name] = "numeric"
+                dataset.type_collections["numeric"].append(col_name)
+
+    return dataset