Merge pull request #169 from scikit-learn-contrib/SandroCasagrande-fit_transform

Added fit_transform method to DataFrameMapper

Author: dukebody

README.rst

@@ -408,6 +408,10 @@ Example: imputing with a fixed value:
 
 Changelog
 ---------
+Unreleased
+**********
+* Change behaviour of DataFrameMapper's fit_transform method to invoke each underlying transformers'
+  native fit_transform if implemented. (#150)
 
 1.7.0 (2018-08-15)
 ******************
@@ -417,7 +421,6 @@ Changelog
   with values other than the mode (#144).
 * Preserve input data types when no transform is supplied (#138).
 
-
 1.6.0 (2017-10-28)
 ******************
 * Add column name to exception during fit/transform (#110).

sklearn_pandas/dataframe_mapper.py

@@ -114,6 +114,16 @@ def __init__(self, features, default=False, sparse=False, df_out=False,
         if (df_out and (sparse or default)):
             raise ValueError("Can not use df_out with sparse or default")
 
+    def _build(self):
+        """
+        Build attributes built_features and built_default.
+        """
+        if isinstance(self.features, list):
+            self.built_features = [_build_feature(*f) for f in self.features]
+        else:
+            self.built_features = self.features
+        self.built_default = _build_transformer(self.default)
+
     @property
     def _selected_columns(self):
         """
@@ -198,12 +208,7 @@ def fit(self, X, y=None):
         y       the target vector relative to X, optional
 
         """
-        if isinstance(self.features, list):
-            self.built_features = [_build_feature(*f) for f in self.features]
-        else:
-            self.built_features = self.features
-
-        self.built_default = _build_transformer(self.default)
+        self._build()
 
         for columns, transformers, options in self.built_features:
             input_df = options.get('input_df', self.input_df)
@@ -273,23 +278,32 @@ def get_dtype(self, ex):
         else:
             raise TypeError(type(ex))
 
-    def transform(self, X):
+    def _transform(self, X, y=None, do_fit=False):
         """
-        Transform the given data. Assumes that fit has already been called.
-
-        X       the data to transform
+        Transform the given data with possibility to fit in advance.
+        Avoids code duplication for implementation of transform and
+        fit_transform.
         """
+        if do_fit:
+            self._build()
+
         extracted = []
         self.transformed_names_ = []
         for columns, transformers, options in self.built_features:
             input_df = options.get('input_df', self.input_df)
+
             # columns could be a string or list of
             # strings; we don't care because pandas
             # will handle either.
             Xt = self._get_col_subset(X, columns, input_df)
             if transformers is not None:
                 with add_column_names_to_exception(columns):
-                    Xt = transformers.transform(Xt)
+                    if do_fit and hasattr(transformers, 'fit_transform'):
+                        Xt = _call_fit(transformers.fit_transform, Xt, y)
+                    else:
+                        if do_fit:
+                            _call_fit(transformers.fit, Xt, y)
+                        Xt = transformers.transform(Xt)
             extracted.append(_handle_feature(Xt))
 
             alias = options.get('alias')
@@ -302,7 +316,12 @@ def transform(self, X):
             Xt = self._get_col_subset(X, unsel_cols, self.input_df)
             if self.built_default is not None:
                 with add_column_names_to_exception(unsel_cols):
-                    Xt = self.built_default.transform(Xt)
+                    if do_fit and hasattr(self.built_default, 'fit_transform'):
+                        Xt = _call_fit(self.built_default.fit_transform, Xt, y)
+                    else:
+                        if do_fit:
+                            _call_fit(self.built_default.fit, Xt, y)
+                        Xt = self.built_default.transform(Xt)
                 self.transformed_names_ += self.get_names(
                     unsel_cols, self.built_default, Xt)
             else:
@@ -348,3 +367,22 @@ def transform(self, X):
             return df_out
         else:
             return stacked
+
+    def transform(self, X):
+        """
+        Transform the given data. Assumes that fit has already been called.
+
+        X       the data to transform
+        """
+        return self._transform(X)
+
+    def fit_transform(self, X, y=None):
+        """
+        Fit a transformation from the pipeline and directly apply
+        it to the given data.
+
+        X       the data to fit
+
+        y       the target vector relative to X, optional
+        """
+        return self._transform(X, y, True)

tests/test_dataframe_mapper.py

@@ -402,6 +402,50 @@ def test_pca(complex_dataframe):
     assert cols[1] == 'feat1_feat2_1'
 
 
+def test_fit_transform(simple_dataframe):
+    """
+    Check that custom fit_transform methods of the transformers are invoked.
+    """
+    df = simple_dataframe
+    mock_transformer = Mock()
+    # return something of measurable length but does nothing
+    mock_transformer.fit_transform.return_value = np.array([1, 2, 3])
+    mapper = DataFrameMapper([("a", mock_transformer)])
+    mapper.fit_transform(df)
+    assert mock_transformer.fit_transform.called
+
+
+def test_fit_transform_equiv_mock(simple_dataframe):
+    """
+    Check for equivalent results for code paths fit_transform
+    versus fit and transform in DataFrameMapper using the mock
+    transformer which does not implement a custom fit_transform.
+    """
+    df = simple_dataframe
+    mapper = DataFrameMapper([('a', MockXTransformer())])
+    transformed_combined = mapper.fit_transform(df)
+    transformed_separate = mapper.fit(df).transform(df)
+    assert np.all(transformed_combined == transformed_separate)
+
+
+def test_fit_transform_equiv_pca(complex_dataframe):
+    """
+    Check for equivalent results for code paths fit_transform
+    versus fit and transform in DataFrameMapper and transformer
+    using PCA which implements a custom fit_transform. The
+    equivalence of both paths in the transformer only can be
+    asserted since this is tested in the sklearn tests
+    scikit-learn/sklearn/decomposition/tests/test_pca.py
+    """
+    df = complex_dataframe
+    mapper = DataFrameMapper(
+        [(['feat1', 'feat2'], sklearn.decomposition.PCA(2))],
+        df_out=True)
+    transformed_combined = mapper.fit_transform(df)
+    transformed_separate = mapper.fit(df).transform(df)
+    assert np.allclose(transformed_combined, transformed_separate)
+
+
 def test_input_df_true_first_transformer(simple_dataframe, monkeypatch):
     """
     If input_df is True, the first transformer is passed
@@ -438,7 +482,8 @@ def test_input_df_true_next_transformers(simple_dataframe, monkeypatch):
     mapper = DataFrameMapper([
         ('a', [MockXTransformer(), MockTClassifier()])
     ], input_df=True)
-    out = mapper.fit_transform(df)
+    mapper.fit(df)
+    out = mapper.transform(df)
 
     args, _ = MockTClassifier().fit.call_args
     assert isinstance(args[0], pd.Series)
@@ -537,15 +582,14 @@ def test_get_col_subset_single_column_list(simple_dataframe):
 
 def test_cols_string_array(simple_dataframe):
     """
-    If an string specified as the columns, the transformer
+    If a string is specified as the columns, the transformer
     is called with a 1-d array as input.
     """
     df = simple_dataframe
     mock_transformer = Mock()
-    mock_transformer.transform.return_value = np.array([1, 2, 3])  # do nothing
     mapper = DataFrameMapper([("a", mock_transformer)])
 
-    mapper.fit_transform(df)
+    mapper.fit(df)
     args, kwargs = mock_transformer.fit.call_args
     assert args[0].shape == (3,)
 
@@ -557,10 +601,9 @@ def test_cols_list_column_vector(simple_dataframe):
     """
     df = simple_dataframe
     mock_transformer = Mock()
-    mock_transformer.transform.return_value = np.array([1, 2, 3])  # do nothing
     mapper = DataFrameMapper([(["a"], mock_transformer)])
 
-    mapper.fit_transform(df)
+    mapper.fit(df)
     args, kwargs = mock_transformer.fit.call_args
     assert args[0].shape == (3, 1)