[Feat] Better traditional pipeline cutoff time (#141)

* [Feat] Better traditional pipeline cutoff time

* Fix unit testing

* Better failure msg

* bug fix catboost

* Feedback from Ravin

* First batch of feedback from comments

* Missed examples

* Syntax fix

Author: franchuterivera

autoPyTorch/api/base_task.py

@@ -122,8 +122,8 @@ def search(
         budget_type: Optional[str] = None,
         budget: Optional[float] = None,
         total_walltime_limit: int = 100,
-        func_eval_time_limit: int = 60,
-        traditional_per_total_budget: float = 0.1,
+        func_eval_time_limit_secs: Optional[int] = None,
+        enable_traditional_pipeline: bool = True,
         memory_limit: Optional[int] = 4096,
         smac_scenario_args: Optional[Dict[str, Any]] = None,
         get_smac_object_callback: Optional[Callable] = None,
@@ -156,16 +156,24 @@ def search(
                 in seconds for the search of appropriate models.
                 By increasing this value, autopytorch has a higher
                 chance of finding better models.
-            func_eval_time_limit (int), (default=60): Time limit
+            func_eval_time_limit_secs (int), (default=None): Time limit
                 for a single call to the machine learning model.
                 Model fitting will be terminated if the machine
                 learning algorithm runs over the time limit. Set
                 this value high enough so that typical machine
                 learning algorithms can be fit on the training
                 data.
-            traditional_per_total_budget (float), (default=0.1):
-                Percent of total walltime to be allocated for
-                running traditional classifiers.
+                When set to None, this time will automatically be set to
+                total_walltime_limit // 2 to allow enough time to fit
+                at least 2 individual machine learning algorithms.
+                Set to np.inf in case no time limit is desired.
+            enable_traditional_pipeline (bool), (default=True):
+                We fit traditional machine learning algorithms
+                (LightGBM, CatBoost, RandomForest, ExtraTrees, KNN, SVM)
+                before building PyTorch Neural Networks. You can disable this
+                feature by turning this flag to False. All machine learning
+                algorithms that are fitted during search() are considered for
+                ensemble building.
             memory_limit (Optional[int]), (default=4096): Memory
                 limit in MB for the machine learning algorithm. autopytorch
                 will stop fitting the machine learning algorithm if it tries
@@ -228,8 +236,8 @@ def search(
             budget_type=budget_type,
             budget=budget,
             total_walltime_limit=total_walltime_limit,
-            func_eval_time_limit=func_eval_time_limit,
-            traditional_per_total_budget=traditional_per_total_budget,
+            func_eval_time_limit_secs=func_eval_time_limit_secs,
+            enable_traditional_pipeline=enable_traditional_pipeline,
             memory_limit=memory_limit,
             smac_scenario_args=smac_scenario_args,
             get_smac_object_callback=get_smac_object_callback,

autoPyTorch/api/tabular_classification.py

@@ -103,26 +103,27 @@ def _get_required_dataset_properties(self, dataset: BaseDataset) -> Dict[str, An
     def build_pipeline(self, dataset_properties: Dict[str, Any]) -> TabularRegressionPipeline:
         return TabularRegressionPipeline(dataset_properties=dataset_properties)
 
-    def search(self,
-               optimize_metric: str,
-               X_train: Optional[Union[List, pd.DataFrame, np.ndarray]] = None,
-               y_train: Optional[Union[List, pd.DataFrame, np.ndarray]] = None,
-               X_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None,
-               y_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None,
-               dataset_name: Optional[str] = None,
-               budget_type: Optional[str] = None,
-               budget: Optional[float] = None,
-               total_walltime_limit: int = 100,
-               func_eval_time_limit: int = 60,
-               traditional_per_total_budget: float = 0.1,
-               memory_limit: Optional[int] = 4096,
-               smac_scenario_args: Optional[Dict[str, Any]] = None,
-               get_smac_object_callback: Optional[Callable] = None,
-               all_supported_metrics: bool = True,
-               precision: int = 32,
-               disable_file_output: List = [],
-               load_models: bool = True,
-               ) -> 'BaseTask':
+    def search(
+        self,
+        optimize_metric: str,
+        X_train: Optional[Union[List, pd.DataFrame, np.ndarray]] = None,
+        y_train: Optional[Union[List, pd.DataFrame, np.ndarray]] = None,
+        X_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None,
+        y_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None,
+        dataset_name: Optional[str] = None,
+        budget_type: Optional[str] = None,
+        budget: Optional[float] = None,
+        total_walltime_limit: int = 100,
+        func_eval_time_limit_secs: Optional[int] = None,
+        enable_traditional_pipeline: bool = False,
+        memory_limit: Optional[int] = 4096,
+        smac_scenario_args: Optional[Dict[str, Any]] = None,
+        get_smac_object_callback: Optional[Callable] = None,
+        all_supported_metrics: bool = True,
+        precision: int = 32,
+        disable_file_output: List = [],
+        load_models: bool = True,
+    ) -> 'BaseTask':
         """
         Search for the best pipeline configuration for the given dataset.
 
@@ -147,16 +148,20 @@ def search(self,
                 in seconds for the search of appropriate models.
                 By increasing this value, autopytorch has a higher
                 chance of finding better models.
-            func_eval_time_limit (int), (default=60): Time limit
+            func_eval_time_limit_secs (int), (default=None): Time limit
                 for a single call to the machine learning model.
                 Model fitting will be terminated if the machine
                 learning algorithm runs over the time limit. Set
                 this value high enough so that typical machine
                 learning algorithms can be fit on the training
                 data.
-            traditional_per_total_budget (float), (default=0.1):
-                Percent of total walltime to be allocated for
-                running traditional classifiers.
+                When set to None, this time will automatically be set to
+                total_walltime_limit // 2 to allow enough time to fit
+                at least 2 individual machine learning algorithms.
+                Set to np.inf in case no time limit is desired.
+            enable_traditional_pipeline (bool), (default=False):
+                Not enabled for regression. This flag is here to comply
+                with the API.
             memory_limit (Optional[int]), (default=4096): Memory
                 limit in MB for the machine learning algorithm. autopytorch
                 will stop fitting the machine learning algorithm if it tries
@@ -219,8 +224,8 @@ def search(self,
             budget_type=budget_type,
             budget=budget,
             total_walltime_limit=total_walltime_limit,
-            func_eval_time_limit=func_eval_time_limit,
-            traditional_per_total_budget=traditional_per_total_budget,
+            func_eval_time_limit_secs=func_eval_time_limit_secs,
+            enable_traditional_pipeline=enable_traditional_pipeline,
             memory_limit=memory_limit,
             smac_scenario_args=smac_scenario_args,
             get_smac_object_callback=get_smac_object_callback,

autoPyTorch/api/tabular_regression.py

@@ -377,8 +377,9 @@ def _loss(self, y_true: np.ndarray, y_hat: np.ndarray) -> Dict[str, float]:
 
         """
 
-        if not isinstance(self.configuration, Configuration):
-            return {self.metric.name: self.metric._worst_possible_result}
+        if isinstance(self.configuration, int):
+            # We do not calculate performance of the dummy configurations
+            return {self.metric.name: self.metric._optimum - self.metric._sign * self.metric._worst_possible_result}
 
         if self.additional_metrics is not None:
             metrics = self.additional_metrics

autoPyTorch/evaluation/abstract_evaluator.py

@@ -442,7 +442,7 @@ def run(
 
         empty_queue(queue)
         self.logger.debug(
-            'Finished function evaluation. Status: %s, Cost: %f, Runtime: %f, Additional %s',
-            status, cost, runtime, additional_run_info,
+            'Finished function evaluation %s. Status: %s, Cost: %f, Runtime: %f, Additional %s',
+            str(num_run), status, cost, runtime, additional_run_info,
         )
         return status, cost, runtime, additional_run_info

autoPyTorch/evaluation/tae.py

@@ -110,6 +110,10 @@ def fit_predict_and_loss(self) -> None:
 
             status = StatusType.SUCCESS
 
+            self.logger.debug("In train evaluator fit_predict_and_loss, num_run: {} loss:{}".format(
+                self.num_run,
+                loss
+            ))
             self.finish_up(
                 loss=loss,
                 train_loss=train_loss,
@@ -236,7 +240,10 @@ def fit_predict_and_loss(self) -> None:
             self.pipeline = self._get_pipeline()
 
             status = StatusType.SUCCESS
-            self.logger.debug("In train evaluator fit_predict_and_loss, loss:{}".format(opt_loss))
+            self.logger.debug("In train evaluator fit_predict_and_loss, num_run: {} loss:{}".format(
+                self.num_run,
+                opt_loss
+            ))
             self.finish_up(
                 loss=opt_loss,
                 train_loss=train_loss,

autoPyTorch/evaluation/train_evaluator.py

@@ -84,7 +84,7 @@ def __init__(self,
                  dataset_name: str,
                  backend: Backend,
                  total_walltime_limit: float,
-                 func_eval_time_limit: float,
+                 func_eval_time_limit_secs: float,
                  memory_limit: typing.Optional[int],
                  metric: autoPyTorchMetric,
                  watcher: StopWatch,
@@ -120,7 +120,7 @@ def __init__(self,
                 An interface with disk
             total_walltime_limit (float):
                 The maximum allowed time for this job
-            func_eval_time_limit (float):
+            func_eval_time_limit_secs (float):
                 How much each individual task is allowed to last
             memory_limit (typing.Optional[int]):
                 Maximum allowed CPU memory this task can use
@@ -180,7 +180,7 @@ def __init__(self,
         # and a bunch of useful limits
         self.worst_possible_result = get_cost_of_crash(self.metric)
         self.total_walltime_limit = int(total_walltime_limit)
-        self.func_eval_time_limit = int(func_eval_time_limit)
+        self.func_eval_time_limit_secs = int(func_eval_time_limit_secs)
         self.memory_limit = memory_limit
         self.watcher = watcher
         self.seed = seed
@@ -265,7 +265,7 @@ def run_smbo(self, func: typing.Optional[typing.Callable] = None
         scenario_dict = {
             'abort_on_first_run_crash': False,
             'cs': self.config_space,
-            'cutoff_time': self.func_eval_time_limit,
+            'cutoff_time': self.func_eval_time_limit_secs,
             'deterministic': 'true',
             'instances': instances,
             'memory_limit': self.memory_limit,

autoPyTorch/optimizer/smbo.py

@@ -13,12 +13,22 @@
     SVMModel)
 
 _classifiers = {
+    # Sort by more robust models
+    # Depending on the allocated time budget, only the
+    # top models from this dict are two be fitted.
+    # LGBM is the more robust model, with
+    # internal measures to prevent crashes, overfit
+    # Additionally, it is one of the state of the art
+    # methods for tabular prediction.
+    # Then follow with catboost for categorical heavy
+    # datasets. The other models are complementary and
+    # their ordering is not critical
+    'lgb': LGBModel,
     'catboost': CatboostModel,
+    'random_forest': RFModel,
     'extra_trees': ExtraTreesModel,
+    'svm_classifier': SVMModel,
     'knn_classifier': KNNModel,
-    'lgb': LGBModel,
-    'random_forest': RFModel,
-    'svm_classifier': SVMModel
 }
 _addons = ThirdPartyComponents(BaseClassifier)
 

autoPyTorch/pipeline/components/setup/traditional_ml/classifier_models/__init__.py

@@ -211,8 +211,13 @@ def get_pipeline_representation(self) -> Dict[str, str]:
         """
         estimator_name = 'TraditionalTabularClassification'
         if self.steps[0][1].choice is not None:
-            estimator_name = cast(str,
-                                  self.steps[0][1].choice.model.get_properties()['shortname'])
+            if self.steps[0][1].choice.model is None:
+                estimator_name = self.steps[0][1].choice.__class__.__name__
+            else:
+                estimator_name = cast(
+                    str,
+                    self.steps[0][1].choice.model.get_properties()['shortname']
+                )
         return {
             'Preprocessing': 'None',
             'Estimator': estimator_name,

autoPyTorch/pipeline/traditional_tabular_classification.py

@@ -57,7 +57,7 @@
         y_test=y_test.copy(),
         optimize_metric='accuracy',
         total_walltime_limit=300,
-        func_eval_time_limit=50
+        func_eval_time_limit_secs=50
     )
 
     ############################################################################