Merge pull request #107 from theislab/dev

Dev

Author: davidsebfischer

.gitignore

@@ -14,6 +14,6 @@ resources/*
 */*.ipynb_checkpoints/
 **/.DS_Store
 docs/_templates/
-
+dist/
 !**/.gitignore
 

diffxpy/fit/fit.py

@@ -3,21 +3,21 @@
     from anndata.base import Raw
 except ImportError:
     from anndata import Raw
+import batchglm.api as glm
 import logging
 import numpy as np
 import pandas as pd
 import patsy
 import scipy.sparse
 from typing import Union, List, Dict, Callable, Tuple
 
-from batchglm.models.base import _InputDataBase
 from .external import _fit
 from .external import parse_gene_names, parse_sample_description, parse_size_factors, parse_grouping, \
     constraint_system_from_star
 
 
 def model(
-        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
         formula_loc: Union[None, str] = None,
         formula_scale: Union[None, str] = "~1",
         as_numeric: Union[List[str], Tuple[str], str] = (),
@@ -164,6 +164,10 @@ def model(
 
         Should be "float32" for single precision or "float64" for double precision.
     :param kwargs: [Debugging] Additional arguments will be passed to the _fit method.
+    :return:
+        An estimator instance that contains all estimation relevant attributes and the model in estim.model.
+        The attributes of the model depend on the noise model and the covariates used.
+        We provide documentation for the model class in the model section of the documentation.
     """
     if len(kwargs) != 0:
         logging.getLogger("diffxpy").debug("additional kwargs: %s", str(kwargs))
@@ -222,7 +226,7 @@ def model(
 
 
 def residuals(
-        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
         formula_loc: Union[None, str] = None,
         formula_scale: Union[None, str] = "~1",
         as_numeric: Union[List[str], Tuple[str], str] = (),
@@ -396,7 +400,7 @@ def residuals(
 
 
 def partition(
-        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
         parts: Union[str, np.ndarray, list],
         gene_names: Union[np.ndarray, list] = None,
         sample_description: pd.DataFrame = None,
@@ -450,7 +454,7 @@ class _Partition:
 
     def __init__(
             self,
-            data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+            data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
             parts: Union[str, np.ndarray, list],
             gene_names: Union[np.ndarray, list] = None,
             sample_description: pd.DataFrame = None,
@@ -477,7 +481,7 @@ def __init__(
             same order as in data or string-type column identifier of size-factor containing
             column in sample description.
         """
-        if isinstance(data, _InputDataBase):
+        if isinstance(data, glm.typing.InputDataBaseTyping):
             self.x = data.x
         elif isinstance(data, anndata.AnnData) or isinstance(data, Raw):
             self.x = data.X

diffxpy/testing/det.py

@@ -1,22 +1,18 @@
 import abc
+try:
+    import anndata
+except ImportError:
+    anndata = None
+import batchglm.api as glm
 import logging
-from typing import Union, Dict, Tuple, List, Set
+import numpy as np
+import patsy
 import pandas as pd
 from random import sample
 import scipy.sparse
-
-import numpy as np
-import patsy
+from typing import Union, Dict, Tuple, List, Set
 
 from .utils import split_x, dmat_unique
-
-try:
-    import anndata
-except ImportError:
-    anndata = None
-
-from batchglm.models.base import _EstimatorBase, _InputDataBase
-
 from ..stats import stats
 from . import correction
 from diffxpy import pkg_constants
@@ -468,17 +464,17 @@ class DifferentialExpressionTestLRT(_DifferentialExpressionTestSingle):
 
     sample_description: pd.DataFrame
     full_design_loc_info: patsy.design_info
-    full_estim: _EstimatorBase
+    full_estim: glm.typing.EstimatorBaseTyping
     reduced_design_loc_info: patsy.design_info
-    reduced_estim: _EstimatorBase
+    reduced_estim: glm.typing.EstimatorBaseTyping
 
     def __init__(
             self,
             sample_description: pd.DataFrame,
             full_design_loc_info: patsy.design_info,
-            full_estim: _EstimatorBase,
+            full_estim: glm.typing.EstimatorBaseTyping,
             reduced_design_loc_info: patsy.design_info,
-            reduced_estim: _EstimatorBase
+            reduced_estim: glm.typing.EstimatorBaseTyping
     ):
         super().__init__()
         self.sample_description = sample_description
@@ -689,7 +685,7 @@ class DifferentialExpressionTestWald(_DifferentialExpressionTestSingle):
     Single wald test per gene.
     """
 
-    model_estim: _EstimatorBase
+    model_estim: glm.typing.EstimatorBaseTyping
     sample_description: pd.DataFrame
     coef_loc_totest: np.ndarray
     theta_mle: np.ndarray
@@ -699,7 +695,7 @@ class DifferentialExpressionTestWald(_DifferentialExpressionTestSingle):
 
     def __init__(
             self,
-            model_estim: _EstimatorBase,
+            model_estim: glm.typing.EstimatorBaseTyping,
             col_indices: np.ndarray,
             noise_model: str,
             sample_description: pd.DataFrame
@@ -1548,7 +1544,7 @@ def __init__(
         super().__init__()
         if isinstance(data, anndata.AnnData) or isinstance(data, anndata.Raw):
             data = data.X
-        elif isinstance(data, _InputDataBase):
+        elif isinstance(data, glm.typing.InputDataBaseTyping):
             data = data.x
         self._x = data
         self.sample_description = sample_description
@@ -1673,7 +1669,7 @@ def __init__(
         super().__init__()
         if isinstance(data, anndata.AnnData) or isinstance(data, anndata.Raw):
             data = data.X
-        elif isinstance(data, _InputDataBase):
+        elif isinstance(data, glm.typing.InputDataBaseTyping):
             data = data.x
         self._x = data
         self.sample_description = sample_description
@@ -2090,13 +2086,13 @@ class DifferentialExpressionTestZTest(_DifferentialExpressionTestMulti):
     Pairwise unit_test between more than 2 groups per gene.
     """
 
-    model_estim: _EstimatorBase
+    model_estim: glm.typing.EstimatorBaseTyping
     theta_mle: np.ndarray
     theta_sd: np.ndarray
 
     def __init__(
             self,
-            model_estim: _EstimatorBase,
+            model_estim: glm.typing.EstimatorBaseTyping,
             grouping,
             groups,
             correction_type: str
@@ -2293,13 +2289,13 @@ class DifferentialExpressionTestZTestLazy(_DifferentialExpressionTestMulti):
     memory.
     """
 
-    model_estim: _EstimatorBase
+    model_estim: glm.typing.EstimatorBaseTyping
     _theta_mle: np.ndarray
     _theta_sd: np.ndarray
 
     def __init__(
             self,
-            model_estim: _EstimatorBase,
+            model_estim: glm.typing.EstimatorBaseTyping,
             grouping, groups,
             correction_type="global"
     ):
@@ -2856,15 +2852,15 @@ def summary(self, qval_thres=None, fc_upper_thres=None,
 
 class _DifferentialExpressionTestCont(_DifferentialExpressionTestSingle):
     _de_test: _DifferentialExpressionTestSingle
-    _model_estim: _EstimatorBase
+    _model_estim: glm.typing.EstimatorBaseTyping
     _size_factors: np.ndarray
     _continuous_coords: np.ndarray
     _spline_coefs: list
 
     def __init__(
             self,
             de_test: _DifferentialExpressionTestSingle,
-            model_estim: _EstimatorBase,
+            model_estim: glm.typing.EstimatorBaseTyping,
             size_factors: np.ndarray,
             continuous_coords: np.ndarray,
             spline_coefs: list,

diffxpy/testing/tests.py

@@ -3,15 +3,14 @@
     from anndata.base import Raw
 except ImportError:
     from anndata import Raw
+import batchglm.api as glm
 import logging
 import numpy as np
 import pandas as pd
 import patsy
 import scipy.sparse
 from typing import Union, List, Dict, Callable, Tuple
 
-from batchglm import data as data_utils
-from batchglm.models.base import _EstimatorBase, _InputDataBase
 from diffxpy import pkg_constants
 from diffxpy.models.batch_bfgs.optim import Estim_BFGS
 from .det import DifferentialExpressionTestLRT, DifferentialExpressionTestWald, \
@@ -40,7 +39,7 @@ def _fit(
         quick_scale: bool = None,
         close_session=True,
         dtype="float64"
-) -> _EstimatorBase:
+) -> glm.typing.InputDataBaseTyping:
     """
     :param noise_model: str, noise model to use in model-based unit_test. Possible options:
 
@@ -187,7 +186,7 @@ def _fit(
 
 
 def lrt(
-        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
         full_formula_loc: str,
         reduced_formula_loc: str,
         full_formula_scale: str = "~1",
@@ -298,25 +297,25 @@ def lrt(
         sample_description=sample_description
     )
 
-    full_design_loc = data_utils.design_matrix(
+    full_design_loc = glm.data.design_matrix(
         sample_description=sample_description,
         formula=full_formula_loc,
         as_categorical=[False if x in as_numeric else True for x in sample_description.columns.values],
         return_type="patsy"
     )
-    reduced_design_loc = data_utils.design_matrix(
+    reduced_design_loc = glm.data.design_matrix(
         sample_description=sample_description,
         formula=reduced_formula_loc,
         as_categorical=[False if x in as_numeric else True for x in sample_description.columns.values],
         return_type="patsy"
     )
-    full_design_scale = data_utils.design_matrix(
+    full_design_scale = glm.data.design_matrix(
         sample_description=sample_description,
         formula=full_formula_scale,
         as_categorical=[False if x in as_numeric else True for x in sample_description.columns.values],
         return_type="patsy"
     )
-    reduced_design_scale = data_utils.design_matrix(
+    reduced_design_scale = glm.data.design_matrix(
         sample_description=sample_description,
         formula=reduced_formula_scale,
         as_categorical=[False if x in as_numeric else True for x in sample_description.columns.values],
@@ -371,7 +370,7 @@ def lrt(
 
 
 def wald(
-        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
         factor_loc_totest: Union[str, List[str]] = None,
         coef_to_test: Union[str, List[str]] = None,
         formula_loc: Union[None, str] = None,
@@ -597,7 +596,7 @@ def wald(
     elif coef_to_test is not None:
         # Directly select coefficients to test from design matrix (xarray):
         # Check that coefficients to test are not dependent parameters if constraints are given:
-        coef_loc_names = data_utils.view_coef_names(design_loc).tolist()
+        coef_loc_names = glm.data.view_coef_names(design_loc).tolist()
         if not np.all([x in coef_loc_names for x in coef_to_test]):
             raise ValueError(
                 "the requested test coefficients %s were found in model coefficients %s" %
@@ -645,7 +644,7 @@ def wald(
 
 
 def t_test(
-        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
         grouping,
         gene_names: Union[np.ndarray, list] = None,
         sample_description: pd.DataFrame = None,
@@ -687,7 +686,7 @@ def t_test(
 
 
 def rank_test(
-        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
         grouping: Union[str, np.ndarray, list],
         gene_names: Union[np.ndarray, list] = None,
         sample_description: pd.DataFrame = None,
@@ -729,7 +728,7 @@ def rank_test(
 
 
 def two_sample(
-        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
         grouping: Union[str, np.ndarray, list],
         as_numeric: Union[List[str], Tuple[str], str] = (),
         test: str = "t-test",
@@ -902,7 +901,7 @@ def two_sample(
 
 
 def pairwise(
-        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
         grouping: Union[str, np.ndarray, list],
         as_numeric: Union[List[str], Tuple[str], str] = (),
         test: str = 'z-test',
@@ -1026,7 +1025,7 @@ def pairwise(
 
     if test.lower() == 'z-test' or test.lower() == 'z_test' or test.lower() == 'ztest':
         # -1 in formula removes intercept
-        dmat = data_utils.design_matrix(
+        dmat = glm.data.design_matrix(
             sample_description,
             formula="~ 1 - 1 + grouping"
         )
@@ -1113,7 +1112,7 @@ def pairwise(
 
 
 def versus_rest(
-        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
         grouping: Union[str, np.ndarray, list],
         as_numeric: Union[List[str], Tuple[str], str] = (),
         test: str = 'wald',
@@ -1275,7 +1274,7 @@ def versus_rest(
 
 
 def partition(
-        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+        data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
         parts: Union[str, np.ndarray, list],
         gene_names: Union[np.ndarray, list] = None,
         sample_description: pd.DataFrame = None
@@ -1318,7 +1317,7 @@ class _Partition:
 
     def __init__(
             self,
-            data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, _InputDataBase],
+            data: Union[anndata.AnnData, Raw, np.ndarray, scipy.sparse.csr_matrix, glm.typing.InputDataBaseTyping],
             parts: Union[str, np.ndarray, list],
             gene_names: Union[np.ndarray, list] = None,
             sample_description: pd.DataFrame = None
@@ -1333,7 +1332,7 @@ def __init__(
         :param gene_names: optional list/array of gene names which will be used if `data` does not implicitly store these
         :param sample_description: optional pandas.DataFrame containing sample annotations
         """
-        if isinstance(data, _InputDataBase):
+        if isinstance(data, glm.typing.InputDataBaseTyping):
             self.x = data.x
         elif isinstance(data, anndata.AnnData) or isinstance(data, Raw):
             self.x = data.X