From 21c80296df6a337b78a4358f4bf7e524e795245d Mon Sep 17 00:00:00 2001 From: Rhushil Vasavada Date: Sun, 1 Jun 2025 12:50:40 -0500 Subject: [PATCH 01/27] Trying out multioutput PCovC --- src/skmatter/decomposition/_pcovc.py | 77 ++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 11 deletions(-) diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index e0cee034e..4e209fdf1 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -10,6 +10,7 @@ SGDClassifier, ) from sklearn.linear_model._base import LinearClassifierMixin +from sklearn.multioutput import MultiOutputClassifier from sklearn.svm import LinearSVC from sklearn.utils import check_array from sklearn.utils.multiclass import check_classification_targets, type_of_target @@ -258,7 +259,7 @@ def fit(self, X, Y, W=None): not passed, it is assumed that the weights will be taken from a linear classifier fit between :math:`\mathbf{X}` and :math:`\mathbf{Y}` """ - X, Y = validate_data(self, X, Y, y_numeric=False) + X, Y = validate_data(self, X, Y, multi_output=True, y_numeric=False) check_classification_targets(Y) self.classes_ = np.unique(Y) @@ -269,6 +270,7 @@ def fit(self, X, Y, W=None): LogisticRegressionCV, LinearSVC, LinearDiscriminantAnalysis, + MultiOutputClassifier, RidgeClassifier, RidgeClassifierCV, SGDClassifier, @@ -285,22 +287,39 @@ def fit(self, X, Y, W=None): ) if self.classifier != "precomputed": - if self.classifier is None: + if self.classifier is None and Y.ndim < 2: classifier = LogisticRegression() + elif self.classifier is None and Y.ndim >= 2: + classifier = MultiOutputClassifier(estimator=LogisticRegression()) else: classifier = self.classifier self.z_classifier_ = check_cl_fit(classifier, X, Y) - W = self.z_classifier_.coef_.T + + if isinstance(self.z_classifier_, MultiOutputClassifier): + W = np.hstack([est_.coef_.T for est_ in self.z_classifier_.estimators_]) + print(W.shape) + else: + W = self.z_classifier_.coef_.T.reshape(X.shape[1], -1) else: # If precomputed, use default classifier to predict Y from T - classifier = LogisticRegression() - if W is None: - W = LogisticRegression().fit(X, Y).coef_.T + # check for the case of 2D Y -- we need to make sure that this is MultiOutputClassifier instead + if Y.ndim >= 2: + classifier = MultiOutputClassifier(estimator=LogisticRegression) + if W is None: + _ = MultiOutputClassifier(estimator=LogisticRegression).fit(X, Y) + W = np.hstack([est_.coef_.T for est_ in _.estimators_]) + else: + classifier = LogisticRegression() + if W is None: + W = LogisticRegression().fit(X, Y).coef_.T + W = W.reshape(X.shape[1], -1) + # print(f"X {X.shape}") + # print(f"W {W.shape}") Z = X @ W - + # print(f"Z {Z.shape}") if self.space_ == "feature": self._fit_feature_space(X, Y, Z) else: @@ -310,8 +329,19 @@ def fit(self, X, Y, W=None): # classifier and steal weights to get pxz and ptz self.classifier_ = clone(classifier).fit(X @ self.pxt_, Y) - self.ptz_ = self.classifier_.coef_.T - self.pxz_ = self.pxt_ @ self.ptz_ + if isinstance(self.classifier_, MultiOutputClassifier): + self.ptz_ = np.hstack( + [est_.coef_.T for est_ in self.classifier_.estimators_] + ) + # print(f"pxt {self.pxt_.shape}") + # print(f"ptz {self.ptz_.shape}") + self.pxz_ = self.pxt_ @ self.ptz_ + # print(f"pxz {self.pxz_.shape}") + + else: + self.ptz_ = self.classifier_.coef_.T + # print(self.ptz_.shape) + self.pxz_ = self.pxt_ @ self.ptz_ if len(Y.shape) == 1 and type_of_target(Y) == "binary": self.pxz_ = self.pxz_.reshape( @@ -422,10 +452,35 @@ def decision_function(self, X=None, T=None): if X is not None: X = validate_data(self, X, reset=False) # Or self.classifier_.decision_function(X @ self.pxt_) - return X @ self.pxz_ + self.classifier_.intercept_ + Z = X @ self.pxz_ else: T = check_array(T) - return T @ self.ptz_ + self.classifier_.intercept_ + Z = T @ self.ptz_ + + if isinstance(self.classifier_, MultiOutputClassifier): + + n_outputs = len(self.classifier_.estimators_) + n_classes = Z.shape[1] // n_outputs + print(Z.shape) + # unpack to 3d + Z = Z.reshape(Z.shape[0], n_outputs, n_classes) + print(Z.shape) + + # add the intercept for estimator in MultiOutputClassifier + for i, est_ in enumerate(self.classifier_.estimators_): + # print(Z[:, i, :][0, :]) + Z[:, i, :] += est_.intercept_ + # print(Z) + # print() + print(est_.intercept_) + # print() + # print(Z[:, i, :][0, :]) + # swap order of Z axesfrom (n_samples, n_outputs, n_classes) to (n_samples, n_classes, n_outputs) as in paper + + return Z.transpose(0, 2, 1) + + print(self.classifier_.intercept_) + return Z + self.classifier_.intercept_ def predict(self, X=None, T=None): """Predicts the property labels using classification on T.""" From 87a3091b88da843bb1bc4aff457efa9442716bfb Mon Sep 17 00:00:00 2001 From: Rhushil Vasavada Date: Wed, 4 Jun 2025 14:46:15 -0500 Subject: [PATCH 02/27] Furthering multioutput support for decision_function --- src/skmatter/decomposition/_pcovc.py | 77 ++++++++++++---------------- src/skmatter/utils/_pcovc_utils.py | 46 ++++++++++------- tests/test_pcovc.py | 2 +- 3 files changed, 62 insertions(+), 63 deletions(-) diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index 4e209fdf1..58e060946 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -287,10 +287,11 @@ def fit(self, X, Y, W=None): ) if self.classifier != "precomputed": - if self.classifier is None and Y.ndim < 2: - classifier = LogisticRegression() - elif self.classifier is None and Y.ndim >= 2: - classifier = MultiOutputClassifier(estimator=LogisticRegression()) + if self.classifier is None: + if Y.ndim < 2: + classifier = LogisticRegression() + else: + classifier = MultiOutputClassifier(estimator=LogisticRegression()) else: classifier = self.classifier @@ -298,28 +299,28 @@ def fit(self, X, Y, W=None): if isinstance(self.z_classifier_, MultiOutputClassifier): W = np.hstack([est_.coef_.T for est_ in self.z_classifier_.estimators_]) - print(W.shape) else: W = self.z_classifier_.coef_.T.reshape(X.shape[1], -1) else: - # If precomputed, use default classifier to predict Y from T - # check for the case of 2D Y -- we need to make sure that this is MultiOutputClassifier instead - if Y.ndim >= 2: - classifier = MultiOutputClassifier(estimator=LogisticRegression) - if W is None: - _ = MultiOutputClassifier(estimator=LogisticRegression).fit(X, Y) - W = np.hstack([est_.coef_.T for est_ in _.estimators_]) - else: + if Y.ndim < 2: + # if self.classifier = "precomputed", use default classifier to predict Y from T classifier = LogisticRegression() if W is None: W = LogisticRegression().fit(X, Y).coef_.T W = W.reshape(X.shape[1], -1) - # print(f"X {X.shape}") - # print(f"W {W.shape}") + else: + classifier = MultiOutputClassifier(estimator=LogisticRegression()) + if W is None: + _ = MultiOutputClassifier(estimator=LogisticRegression).fit(X, Y) + W = np.hstack([est_.coef_.T for est_ in _.estimators_]) + + print(f"X: {X.shape}") + print(f"W: {len(W), W[0]}") + Z = X @ W - # print(f"Z {Z.shape}") + if self.space_ == "feature": self._fit_feature_space(X, Y, Z) else: @@ -451,36 +452,26 @@ def decision_function(self, X=None, T=None): if X is not None: X = validate_data(self, X, reset=False) + + # this is similar to how MultiOutputClassifier handles predict_proba() if n_outputs > 1 + if isinstance(self.classifier_, MultiOutputClassifier): + return [ + est_.decision_function(X @ self.pxt_) + for est_ in self.classifier_.estimators_ + ] + # Or self.classifier_.decision_function(X @ self.pxt_) - Z = X @ self.pxz_ + return X @ self.pxz_ + self.classifier_.intercept_ else: T = check_array(T) - Z = T @ self.ptz_ - - if isinstance(self.classifier_, MultiOutputClassifier): - - n_outputs = len(self.classifier_.estimators_) - n_classes = Z.shape[1] // n_outputs - print(Z.shape) - # unpack to 3d - Z = Z.reshape(Z.shape[0], n_outputs, n_classes) - print(Z.shape) - - # add the intercept for estimator in MultiOutputClassifier - for i, est_ in enumerate(self.classifier_.estimators_): - # print(Z[:, i, :][0, :]) - Z[:, i, :] += est_.intercept_ - # print(Z) - # print() - print(est_.intercept_) - # print() - # print(Z[:, i, :][0, :]) - # swap order of Z axesfrom (n_samples, n_outputs, n_classes) to (n_samples, n_classes, n_outputs) as in paper - - return Z.transpose(0, 2, 1) - - print(self.classifier_.intercept_) - return Z + self.classifier_.intercept_ + + if isinstance(self.classifier_, MultiOutputClassifier): + return [ + est_.decision_function(T @ self.ptz_) + for est_ in self.classifier_.estimators_ + ] + + return T @ self.ptz_ + self.classifier_.intercept_ def predict(self, X=None, T=None): """Predicts the property labels using classification on T.""" diff --git a/src/skmatter/utils/_pcovc_utils.py b/src/skmatter/utils/_pcovc_utils.py index ea55dd60a..3203dca82 100644 --- a/src/skmatter/utils/_pcovc_utils.py +++ b/src/skmatter/utils/_pcovc_utils.py @@ -5,6 +5,8 @@ from sklearn.exceptions import NotFittedError from sklearn.utils.validation import check_is_fitted, validate_data +from sklearn.multioutput import MultiOutputClassifier + def check_cl_fit(classifier, X, y): """ @@ -39,29 +41,35 @@ def check_cl_fit(classifier, X, y): # Check compatibility with X validate_data(fitted_classifier, X, y, reset=False, multi_output=True) - # Check compatibility with the number of features in X and the number of - # classes in y - n_classes = len(np.unique(y)) - - if n_classes == 2: - if fitted_classifier.coef_.shape[0] != 1: - raise ValueError( - "For binary classification, expected classifier coefficients " - "to have shape (1, " - f"{X.shape[1]}) but got shape " - f"{fitted_classifier.coef_.shape}" - ) + # Check coefficent compatibility with the number of features in X and the + # number of classes in y + if isinstance(fitted_classifier, MultiOutputClassifier): + for est_ in fitted_classifier.estimators_: + check_cl_coef(X, est_.coef_, len(est_.classes_)) else: - if fitted_classifier.coef_.shape[0] != n_classes: - raise ValueError( - "For multiclass classification, expected classifier coefficients " - "to have shape " - f"({n_classes}, {X.shape[1]}) but got shape " - f"{fitted_classifier.coef_.shape}" - ) + check_cl_coef(X, fitted_classifier.coef_, len(np.unique(y))) except NotFittedError: fitted_classifier = clone(classifier) fitted_classifier.fit(X, y) return fitted_classifier + + +def check_cl_coef(X, classifier_coef_, n_classes): + if n_classes == 2: + if classifier_coef_.shape[0] != 1: + raise ValueError( + "For binary classification, expected classifier coefficients " + "to have shape (1, " + f"{X.shape[1]}) but got shape " + f"{classifier_coef_.shape}" + ) + else: + if classifier_coef_.shape[0] != n_classes: + raise ValueError( + "For multiclass classification, expected classifier coefficients " + "to have shape " + f"({n_classes}, {X.shape[1]}) but got shape " + f"{classifier_coef_.shape}" + ) diff --git a/tests/test_pcovc.py b/tests/test_pcovc.py index 8607a2e2a..cadf6ad96 100644 --- a/tests/test_pcovc.py +++ b/tests/test_pcovc.py @@ -534,7 +534,7 @@ def test_incompatible_classifier(self): str(cm.exception), "Classifier must be an instance of " "`LogisticRegression`, `LogisticRegressionCV`, `LinearSVC`, " - "`LinearDiscriminantAnalysis`, `RidgeClassifier`, " + "`LinearDiscriminantAnalysis`, `MultiOutputClassifier`, `RidgeClassifier`, " "`RidgeClassifierCV`, `SGDClassifier`, `Perceptron`, " "or `precomputed`", ) From 90277a3e7cfe0e17b1f1b6455f6cfc5d08f62115 Mon Sep 17 00:00:00 2001 From: Rhushil Vasavada Date: Tue, 10 Jun 2025 12:08:53 -0500 Subject: [PATCH 03/27] Starting on docstrings --- src/skmatter/_version.py | 21 ++++++++++++++++ src/skmatter/decomposition/_pcovc.py | 36 ++++++++++++++++++---------- 2 files changed, 44 insertions(+), 13 deletions(-) create mode 100644 src/skmatter/_version.py diff --git a/src/skmatter/_version.py b/src/skmatter/_version.py new file mode 100644 index 000000000..e0c87fad9 --- /dev/null +++ b/src/skmatter/_version.py @@ -0,0 +1,21 @@ +# file generated by setuptools-scm +# don't change, don't track in version control + +__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"] + +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple + from typing import Union + + VERSION_TUPLE = Tuple[Union[int, str], ...] +else: + VERSION_TUPLE = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE + +__version__ = version = '0.2.1.dev62+g7cef97c.d20250618' +__version_tuple__ = version_tuple = (0, 2, 1, 'dev62', 'g7cef97c.d20250618') diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index 58e060946..847233f4c 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -121,8 +121,8 @@ class PCovC(LinearClassifierMixin, _BasePCov): `sklearn.pipeline.Pipeline` with model caching. In such cases, the classifier will be re-fitted on the same training data as the composite estimator. - If None, ``sklearn.linear_model.LogisticRegression()`` - is used as the classifier. + If None and ``Y.ndim < 2``, ``sklearn.linear_model.LogisticRegression()`` is used. + If None and ``Y.ndim == 2``, ``sklearn.multioutput.MultiOutputClassifier()`` is used. iterated_power : int or 'auto', default='auto' Number of iterations for the power method computed by @@ -167,11 +167,13 @@ class PCovC(LinearClassifierMixin, _BasePCov): the projector, or weights, from the input space :math:`\mathbf{X}` to the latent-space projection :math:`\mathbf{T}` - pxz_ : ndarray of size :math:`({n_{features}, })` or :math:`({n_{features}, n_{classes}})` + pxz_ : ndarray of size :math:`({n_{features}, })`, :math:`({n_{features}, n_{classes}})`, \ + or :math:`({n_{components}, n_{classes}*n_{outputs}})` the projector, or weights, from the input space :math:`\mathbf{X}` to the class confidence scores :math:`\mathbf{Z}` - ptz_ : ndarray of size :math:`({n_{components}, })` or :math:`({n_{components}, n_{classes}})` + ptz_ : ndarray of size :math:`({n_{components}, })`, :math:`({n_{components}, n_{classes}})` \ + or :math:`({n_{components}, n_{classes}*n_{outputs}})` the projector, or weights, from the latent-space projection :math:`\mathbf{T}` to the class confidence scores :math:`\mathbf{Z}` @@ -251,13 +253,18 @@ def fit(self, X, Y, W=None): scaled to have unit variance, otherwise :math:`\mathbf{X}` should be scaled so that each feature has a variance of 1 / n_features. - Y : numpy.ndarray, shape (n_samples,) - Training data, where n_samples is the number of samples. + Y : numpy.ndarray, shape (n_samples,) or (n_samples, n_outputs) + Training data, where n_samples is the number of samples and + n_outputs is the number of outputs. If classifier parameter is an instance + of ``sklearn.multioutput.MultiOutputClassifier()``, Y can be of shape + (n_samples, n_outputs). W : numpy.ndarray, shape (n_features, n_classes) Classification weights, optional when classifier is ``precomputed``. If not passed, it is assumed that the weights will be taken from a - linear classifier fit between :math:`\mathbf{X}` and :math:`\mathbf{Y}` + linear classifier fit between :math:`\mathbf{X}` and :math:`\mathbf{Y}`. + In the case of a multioutput classifier ``classifier``, + `` W = np.hstack([est_.coef_.T for est_ in classifier.estimators_])``. """ X, Y = validate_data(self, X, Y, multi_output=True, y_numeric=False) check_classification_targets(Y) @@ -317,7 +324,7 @@ def fit(self, X, Y, W=None): W = np.hstack([est_.coef_.T for est_ in _.estimators_]) print(f"X: {X.shape}") - print(f"W: {len(W), W[0]}") + print(f"W: {W.shape}") Z = X @ W @@ -344,6 +351,7 @@ def fit(self, X, Y, W=None): # print(self.ptz_.shape) self.pxz_ = self.pxt_ @ self.ptz_ + print(self.ptz_.shape) if len(Y.shape) == 1 and type_of_target(Y) == "binary": self.pxz_ = self.pxz_.reshape( X.shape[1], @@ -441,9 +449,12 @@ def decision_function(self, X=None, T=None): Returns ------- - Z : numpy.ndarray, shape (n_samples,) or (n_samples, n_classes) + Z : numpy.ndarray, shape (n_samples,) or (n_samples, n_classes), or a list of \ + n_outputs such arrays if n_outputs > 1 Confidence scores. For binary classification, has shape `(n_samples,)`, - for multiclass classification, has shape `(n_samples, n_classes)` + for multiclass classification, has shape `(n_samples, n_classes)`. If n_outputs > 1, + the list returned can contain such arrays with differing shapes depending on the + number of classes in each output of Y. """ check_is_fitted(self, attributes=["pxz_", "ptz_"]) @@ -464,11 +475,10 @@ def decision_function(self, X=None, T=None): return X @ self.pxz_ + self.classifier_.intercept_ else: T = check_array(T) - + if isinstance(self.classifier_, MultiOutputClassifier): return [ - est_.decision_function(T @ self.ptz_) - for est_ in self.classifier_.estimators_ + est_.decision_function(T) for est_ in self.classifier_.estimators_ ] return T @ self.ptz_ + self.classifier_.intercept_ From a7ea9509d21a34880ec8fc42818c4a2c170ae7ef Mon Sep 17 00:00:00 2001 From: Rhushil Vasavada Date: Sun, 22 Jun 2025 11:34:18 -0500 Subject: [PATCH 04/27] Score function and tests --- src/skmatter/_version.py | 4 +-- src/skmatter/decomposition/_pcovc.py | 40 ++++++++++++++++++++++++++++ tests/test_pcovc.py | 31 +++++++++++++++++++-- 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/src/skmatter/_version.py b/src/skmatter/_version.py index e0c87fad9..db8bdda40 100644 --- a/src/skmatter/_version.py +++ b/src/skmatter/_version.py @@ -17,5 +17,5 @@ __version_tuple__: VERSION_TUPLE version_tuple: VERSION_TUPLE -__version__ = version = '0.2.1.dev62+g7cef97c.d20250618' -__version_tuple__ = version_tuple = (0, 2, 1, 'dev62', 'g7cef97c.d20250618') +__version__ = version = '0.2.1.dev58+gead41e2.d20250623' +__version_tuple__ = version_tuple = (0, 2, 1, 'dev58', 'gead41e2.d20250623') diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index 847233f4c..756c336e5 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -10,6 +10,8 @@ SGDClassifier, ) from sklearn.linear_model._base import LinearClassifierMixin + +from sklearn.base import MultiOutputMixin from sklearn.multioutput import MultiOutputClassifier from sklearn.svm import LinearSVC from sklearn.utils import check_array @@ -19,6 +21,11 @@ from skmatter.utils import check_cl_fit +# No inheritance from MultiOutputMixin because decision_function would fail +# test_check_estimator.py 'check_classifier_multioutput' (line 2479 of estimator_checks.py) +# - this is the only test for MultiOutputClassifiers, so is it OK to exclude this tag? + + class PCovC(LinearClassifierMixin, _BasePCov): r"""Principal Covariates Classification (PCovC). @@ -510,3 +517,36 @@ def transform(self, X=None): and n_features is the number of features. """ return super().transform(X) + + def score(self, X, Y, sample_weight=None): + """Return the accuracy on the given test data and labels. + + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Test samples. + + Y : array-like of shape (n_samples,) or (n_samples, n_outputs) + True labels for `X`. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. Can only be used if the PCovC instance + has been trained on multitarget data. + + Returns + ------- + score : float + Accuracy scores. If the PCovC instance was trained on a 1D Y, + this will call the ``score()`` function defined by + ``sklearn.base.ClassifierMixin``. If trained on a 2D Y, this will + call the ``score()`` function defined by + ``sklearn.multioutput.MultiOutputClassifier``, to ensure multi + """ + X, Y = validate_data(self, X, Y, reset=False) + + if isinstance(self.classifier_, MultiOutputClassifier): + # LinearClassifierMixin.score fails with multioutput-multiclass Y + return self.classifier_.score(X @ self.pxt_, Y) + else: + return self.classifier_.score(X @ self.pxt_, Y, sample_weight=sample_weight) diff --git a/tests/test_pcovc.py b/tests/test_pcovc.py index cadf6ad96..68fb23046 100644 --- a/tests/test_pcovc.py +++ b/tests/test_pcovc.py @@ -3,10 +3,11 @@ import numpy as np from sklearn import exceptions -from sklearn.calibration import LinearSVC from sklearn.datasets import load_iris as get_dataset from sklearn.decomposition import PCA from sklearn.linear_model import LogisticRegression, RidgeClassifier +from sklearn.svm import LinearSVC +from sklearn.multioutput import MultiOutputClassifier from sklearn.naive_bayes import GaussianNB from sklearn.preprocessing import StandardScaler from sklearn.utils.validation import check_X_y @@ -95,8 +96,9 @@ def test_simple_prediction(self): pcovc.fit(self.X, self.Y) Yp = pcovc.predict(self.X) + self.assertLessEqual( - np.linalg.norm(Yp - Yhat) ** 2.0 / np.linalg.norm(Yhat) ** 2.0, + np.linalg.norm(Yp - Yhat) ** 2.0 / np.linalg.norm(Yp) ** 2.0, self.error_tol, ) @@ -576,5 +578,30 @@ def test_incompatible_coef_shape(self): ) +class PCovCMultiOutputTest(PCovCBaseTest): + + def test_projector_shapes(self): + pass + + def test_decision_function(self): + pcovc = PCovC( + classifier=MultiOutputClassifier(LogisticRegression()), n_components=2 + ) + + Y_double = np.column_stack((self.Y, self.Y[::-1])) + pcovc.fit(self.X, Y_double) + + Z = pcovc.decision_function(self.X) + + # list of (n_samples, n_classes) arrays + self.assertEqual(len(Z), Y_double.shape[1]) + + for est, z_slice in zip(pcovc.z_classifier_.estimators_, Z): + with self.subTest(type="z_arrays"): + # each array is shape (n_samples, n_classes) + self.assertEqual(self.X.shape[0], z_slice.shape[0]) + self.assertEqual(est.coef_.shape[0], z_slice.shape[1]) + + if __name__ == "__main__": unittest.main(verbosity=2) From 71cfc19582e9ca4a7556d484031a8c166cf19043 Mon Sep 17 00:00:00 2001 From: Rhushil Vasavada Date: Mon, 23 Jun 2025 13:28:25 -0500 Subject: [PATCH 05/27] Fixing _version.py tracking --- src/skmatter/_version.py | 21 ----- src/skmatter/decomposition/_pcovc.py | 110 +++++++++++++-------------- 2 files changed, 51 insertions(+), 80 deletions(-) delete mode 100644 src/skmatter/_version.py diff --git a/src/skmatter/_version.py b/src/skmatter/_version.py deleted file mode 100644 index db8bdda40..000000000 --- a/src/skmatter/_version.py +++ /dev/null @@ -1,21 +0,0 @@ -# file generated by setuptools-scm -# don't change, don't track in version control - -__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"] - -TYPE_CHECKING = False -if TYPE_CHECKING: - from typing import Tuple - from typing import Union - - VERSION_TUPLE = Tuple[Union[int, str], ...] -else: - VERSION_TUPLE = object - -version: str -__version__: str -__version_tuple__: VERSION_TUPLE -version_tuple: VERSION_TUPLE - -__version__ = version = '0.2.1.dev58+gead41e2.d20250623' -__version_tuple__ = version_tuple = (0, 2, 1, 'dev58', 'gead41e2.d20250623') diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index 756c336e5..890859560 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -174,15 +174,18 @@ class PCovC(LinearClassifierMixin, _BasePCov): the projector, or weights, from the input space :math:`\mathbf{X}` to the latent-space projection :math:`\mathbf{T}` - pxz_ : ndarray of size :math:`({n_{features}, })`, :math:`({n_{features}, n_{classes}})`, \ - or :math:`({n_{components}, n_{classes}*n_{outputs}})` + pxz_ : ndarray of size :math:`({n_{features}, })`, :math:`({n_{features}, n_{classes}})` the projector, or weights, from the input space :math:`\mathbf{X}` - to the class confidence scores :math:`\mathbf{Z}` + to the class confidence scores :math:`\mathbf{Z}`. In the multioutput case, + has shape , :math:`({n_{features}, n_{classes}*n_{outputs}})`, a flattened form + of a 3D tensor. ptz_ : ndarray of size :math:`({n_{components}, })`, :math:`({n_{components}, n_{classes}})` \ or :math:`({n_{components}, n_{classes}*n_{outputs}})` the projector, or weights, from the latent-space projection - :math:`\mathbf{T}` to the class confidence scores :math:`\mathbf{Z}` + :math:`\mathbf{T}` to the class confidence scores :math:`\mathbf{Z}`. + In the multioutput case, has shape , :math:`({n_{components}, n_{classes}*n_{outputs}})`, + a flattened form of a 3D tensor. explained_variance_ : numpy.ndarray of shape (n_components,) The amount of variance explained by each of the selected components. @@ -262,7 +265,7 @@ def fit(self, X, Y, W=None): Y : numpy.ndarray, shape (n_samples,) or (n_samples, n_outputs) Training data, where n_samples is the number of samples and - n_outputs is the number of outputs. If classifier parameter is an instance + n_outputs is the number of outputs. If ``self.classifier`` is an instance of ``sklearn.multioutput.MultiOutputClassifier()``, Y can be of shape (n_samples, n_outputs). @@ -276,6 +279,7 @@ def fit(self, X, Y, W=None): X, Y = validate_data(self, X, Y, multi_output=True, y_numeric=False) check_classification_targets(Y) self.classes_ = np.unique(Y) + self.n_outputs = Y.shape[1] super()._set_fit_params(X) @@ -300,35 +304,23 @@ def fit(self, X, Y, W=None): ", or `precomputed`" ) + # if type_of_target(Y) == "binary" + if self.classifier != "precomputed": if self.classifier is None: - if Y.ndim < 2: - classifier = LogisticRegression() - else: - classifier = MultiOutputClassifier(estimator=LogisticRegression()) + classifier = LogisticRegression() else: classifier = self.classifier self.z_classifier_ = check_cl_fit(classifier, X, Y) - - if isinstance(self.z_classifier_, MultiOutputClassifier): - W = np.hstack([est_.coef_.T for est_ in self.z_classifier_.estimators_]) - else: - W = self.z_classifier_.coef_.T.reshape(X.shape[1], -1) + W = self.z_classifier_.coef_.T.reshape(X.shape[1], -1) else: - if Y.ndim < 2: - # if self.classifier = "precomputed", use default classifier to predict Y from T - classifier = LogisticRegression() - if W is None: - W = LogisticRegression().fit(X, Y).coef_.T - W = W.reshape(X.shape[1], -1) - - else: - classifier = MultiOutputClassifier(estimator=LogisticRegression()) - if W is None: - _ = MultiOutputClassifier(estimator=LogisticRegression).fit(X, Y) - W = np.hstack([est_.coef_.T for est_ in _.estimators_]) + # If precomputed, use default classifier to predict Y from T + classifier = LogisticRegression() + if W is None: + W = LogisticRegression().fit(X, Y).coef_.T + W = W.reshape(X.shape[1], -1) print(f"X: {X.shape}") print(f"W: {W.shape}") @@ -460,7 +452,7 @@ def decision_function(self, X=None, T=None): n_outputs such arrays if n_outputs > 1 Confidence scores. For binary classification, has shape `(n_samples,)`, for multiclass classification, has shape `(n_samples, n_classes)`. If n_outputs > 1, - the list returned can contain such arrays with differing shapes depending on the + the list returned can contain arrays with differing shapes depending on the number of classes in each output of Y. """ check_is_fitted(self, attributes=["pxz_", "ptz_"]) @@ -518,35 +510,35 @@ def transform(self, X=None): """ return super().transform(X) - def score(self, X, Y, sample_weight=None): - """Return the accuracy on the given test data and labels. - - - Parameters - ---------- - X : array-like of shape (n_samples, n_features) - Test samples. - - Y : array-like of shape (n_samples,) or (n_samples, n_outputs) - True labels for `X`. - - sample_weight : array-like of shape (n_samples,), default=None - Sample weights. Can only be used if the PCovC instance - has been trained on multitarget data. - - Returns - ------- - score : float - Accuracy scores. If the PCovC instance was trained on a 1D Y, - this will call the ``score()`` function defined by - ``sklearn.base.ClassifierMixin``. If trained on a 2D Y, this will - call the ``score()`` function defined by - ``sklearn.multioutput.MultiOutputClassifier``, to ensure multi - """ - X, Y = validate_data(self, X, Y, reset=False) - - if isinstance(self.classifier_, MultiOutputClassifier): - # LinearClassifierMixin.score fails with multioutput-multiclass Y - return self.classifier_.score(X @ self.pxt_, Y) - else: - return self.classifier_.score(X @ self.pxt_, Y, sample_weight=sample_weight) + # def score(self, X, Y, sample_weight=None): + # """Return the accuracy on the given test data and labels. Contains support + # for multiclass-multioutput data. + + # Parameters + # ---------- + # X : array-like of shape (n_samples, n_features) + # Test samples. + + # Y : array-like of shape (n_samples,) or (n_samples, n_outputs) + # True labels for `X`. + + # sample_weight : array-like of shape (n_samples,), default=None + # Sample weights. Can only be used if the PCovC instance + # has been trained on single-target data. + + # Returns + # ------- + # score : float + # Accuracy scores. If the PCovC instance was trained on a 1D Y, + # this will call the ``score()`` function defined by + # ``sklearn.base.ClassifierMixin``. If trained on a 2D Y, this will + # call the ``score()`` function defined by + # ``sklearn.multioutput.MultiOutputClassifier``. + # """ + # X, Y = validate_data(self, X, Y, reset=False) + + # if isinstance(self.classifier_, MultiOutputClassifier): + # # LinearClassifierMixin.score fails with multioutput-multiclass Y + # return self.classifier_.score(X @ self.pxt_, Y) + # else: + # return self.classifier_.score(X @ self.pxt_, Y, sample_weight=sample_weight) From febab6d492b47445981de1754c1ff5684387bdbb Mon Sep 17 00:00:00 2001 From: Rhushil Vasavada Date: Wed, 25 Jun 2025 15:42:09 -0500 Subject: [PATCH 06/27] Continuing multiouput work --- src/skmatter/decomposition/_pcovc.py | 18 +++++++++++++----- tests/test_pcovc.py | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index 890859560..f449aa540 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -25,6 +25,8 @@ # test_check_estimator.py 'check_classifier_multioutput' (line 2479 of estimator_checks.py) # - this is the only test for MultiOutputClassifiers, so is it OK to exclude this tag? +# did a search of all classifiers that inherit from MultiOutputMixin - none of them implement +# decision function, so I don't think we need to inherit class PCovC(LinearClassifierMixin, _BasePCov): r"""Principal Covariates Classification (PCovC). @@ -277,9 +279,10 @@ def fit(self, X, Y, W=None): `` W = np.hstack([est_.coef_.T for est_ in classifier.estimators_])``. """ X, Y = validate_data(self, X, Y, multi_output=True, y_numeric=False) + check_classification_targets(Y) self.classes_ = np.unique(Y) - self.n_outputs = Y.shape[1] + self.n_outputs = 1 if Y.ndim == 1 else Y.shape[1] super()._set_fit_params(X) @@ -302,9 +305,15 @@ def fit(self, X, Y, W=None): "Classifier must be an instance of `" f"{'`, `'.join(c.__name__ for c in compatible_classifiers)}`" ", or `precomputed`" - ) + ) + + # if self.n_outputs == 1: + # classifier = LogisticRegression() + # else: + # classifier = MultiOutputClassifier(estimator=LogisticRegression()) - # if type_of_target(Y) == "binary" + # if self.classifier == "precomputed": + if self.classifier != "precomputed": if self.classifier is None: @@ -313,14 +322,13 @@ def fit(self, X, Y, W=None): classifier = self.classifier self.z_classifier_ = check_cl_fit(classifier, X, Y) - W = self.z_classifier_.coef_.T.reshape(X.shape[1], -1) + W = self.z_classifier_.coef_.T else: # If precomputed, use default classifier to predict Y from T classifier = LogisticRegression() if W is None: W = LogisticRegression().fit(X, Y).coef_.T - W = W.reshape(X.shape[1], -1) print(f"X: {X.shape}") print(f"W: {W.shape}") diff --git a/tests/test_pcovc.py b/tests/test_pcovc.py index 68fb23046..232953149 100644 --- a/tests/test_pcovc.py +++ b/tests/test_pcovc.py @@ -3,7 +3,7 @@ import numpy as np from sklearn import exceptions -from sklearn.datasets import load_iris as get_dataset +from sklearn.datasets import load_breast_cancer as get_dataset from sklearn.decomposition import PCA from sklearn.linear_model import LogisticRegression, RidgeClassifier from sklearn.svm import LinearSVC From 880aa656636a609fb63f2980417c07b7c1623cad Mon Sep 17 00:00:00 2001 From: Rhushil Vasavada Date: Fri, 27 Jun 2025 16:30:48 -0500 Subject: [PATCH 07/27] Cleaning things up and adding more tests --- src/skmatter/decomposition/_pcovc.py | 104 +++++++++++++++------------ tests/test_pcovc.py | 68 ++++++++++++++++-- 2 files changed, 120 insertions(+), 52 deletions(-) diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index f449aa540..e83f177ea 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -28,6 +28,7 @@ # did a search of all classifiers that inherit from MultiOutputMixin - none of them implement # decision function, so I don't think we need to inherit + class PCovC(LinearClassifierMixin, _BasePCov): r"""Principal Covariates Classification (PCovC). @@ -178,16 +179,11 @@ class PCovC(LinearClassifierMixin, _BasePCov): pxz_ : ndarray of size :math:`({n_{features}, })`, :math:`({n_{features}, n_{classes}})` the projector, or weights, from the input space :math:`\mathbf{X}` - to the class confidence scores :math:`\mathbf{Z}`. In the multioutput case, - has shape , :math:`({n_{features}, n_{classes}*n_{outputs}})`, a flattened form - of a 3D tensor. + to the class confidence scores :math:`\mathbf{Z}`. - ptz_ : ndarray of size :math:`({n_{components}, })`, :math:`({n_{components}, n_{classes}})` \ - or :math:`({n_{components}, n_{classes}*n_{outputs}})` - the projector, or weights, from the latent-space projection - :math:`\mathbf{T}` to the class confidence scores :math:`\mathbf{Z}`. - In the multioutput case, has shape , :math:`({n_{components}, n_{classes}*n_{outputs}})`, - a flattened form of a 3D tensor. + ptz_ : ndarray of size :math:`({n_{components}, })`, :math:`({n_{components}, n_{classes}})` + the projector, or weights, from from the latent-space projection + :math:`\mathbf{T}` to the class confidence scores :math:`\mathbf{Z}`. explained_variance_ : numpy.ndarray of shape (n_components,) The amount of variance explained by each of the selected components. @@ -279,7 +275,7 @@ def fit(self, X, Y, W=None): `` W = np.hstack([est_.coef_.T for est_ in classifier.estimators_])``. """ X, Y = validate_data(self, X, Y, multi_output=True, y_numeric=False) - + check_classification_targets(Y) self.classes_ = np.unique(Y) self.n_outputs = 1 if Y.ndim == 1 else Y.shape[1] @@ -305,33 +301,51 @@ def fit(self, X, Y, W=None): "Classifier must be an instance of `" f"{'`, `'.join(c.__name__ for c in compatible_classifiers)}`" ", or `precomputed`" - ) + ) - # if self.n_outputs == 1: - # classifier = LogisticRegression() - # else: - # classifier = MultiOutputClassifier(estimator=LogisticRegression()) + if self.n_outputs == 1 and isinstance(self.classifier, MultiOutputClassifier): + raise ValueError( + "Classifier cannot be an instance of `MultiOutputClassifier` when Y is 1D" + ) + + if ( + self.n_outputs != 1 + and self.classifier not in ["precomputed", None] + and not ( + isinstance(self.classifier, MultiOutputClassifier) + or self.classifier == "precomputed" + ) + ): + raise ValueError( + "Classifier must be an instance of `MultiOutputClassifier` when Y is 2D" + ) - # if self.classifier == "precomputed": - + if self.n_outputs == 1: + if self.classifier != "precomputed": + classifier = self.classifier or LogisticRegression() + self.z_classifier_ = check_cl_fit(classifier, X, Y) + W = self.z_classifier_.coef_.T - if self.classifier != "precomputed": - if self.classifier is None: - classifier = LogisticRegression() else: - classifier = self.classifier - - self.z_classifier_ = check_cl_fit(classifier, X, Y) - W = self.z_classifier_.coef_.T + # to be used later on as the classifier fit between T and Y + classifier = LogisticRegression() + if W is None: + W = clone(classifier).fit(X, Y).coef_.T else: - # If precomputed, use default classifier to predict Y from T - classifier = LogisticRegression() - if W is None: - W = LogisticRegression().fit(X, Y).coef_.T + if self.classifier != "precomputed": + classifier = self.classifier or MultiOutputClassifier( + estimator=LogisticRegression() + ) + self.z_classifier_ = check_cl_fit(classifier, X, Y) + W = np.hstack([est_.coef_.T for est_ in self.z_classifier_.estimators_]) - print(f"X: {X.shape}") - print(f"W: {W.shape}") + else: + # to be used later on as the classifier fit between T and Y + classifier = MultiOutputClassifier(estimator=LogisticRegression()) + if W is None: + _ = clone(classifier).fit(X, Y) + W = np.hstack([_.coef_.T for _ in _.estimators_]) Z = X @ W @@ -344,7 +358,11 @@ def fit(self, X, Y, W=None): # classifier and steal weights to get pxz and ptz self.classifier_ = clone(classifier).fit(X @ self.pxt_, Y) - if isinstance(self.classifier_, MultiOutputClassifier): + if self.n_outputs == 1: + self.ptz_ = self.classifier_.coef_.T + # print(self.ptz_.shape) + self.pxz_ = self.pxt_ @ self.ptz_ + else: self.ptz_ = np.hstack( [est_.coef_.T for est_ in self.classifier_.estimators_] ) @@ -353,12 +371,7 @@ def fit(self, X, Y, W=None): self.pxz_ = self.pxt_ @ self.ptz_ # print(f"pxz {self.pxz_.shape}") - else: - self.ptz_ = self.classifier_.coef_.T - # print(self.ptz_.shape) - self.pxz_ = self.pxt_ @ self.ptz_ - - print(self.ptz_.shape) + # print(self.ptz_.shape) if len(Y.shape) == 1 and type_of_target(Y) == "binary": self.pxz_ = self.pxz_.reshape( X.shape[1], @@ -460,7 +473,7 @@ def decision_function(self, X=None, T=None): n_outputs such arrays if n_outputs > 1 Confidence scores. For binary classification, has shape `(n_samples,)`, for multiclass classification, has shape `(n_samples, n_classes)`. If n_outputs > 1, - the list returned can contain arrays with differing shapes depending on the + the list can contain arrays with differing shapes depending on the number of classes in each output of Y. """ check_is_fitted(self, attributes=["pxz_", "ptz_"]) @@ -471,25 +484,24 @@ def decision_function(self, X=None, T=None): if X is not None: X = validate_data(self, X, reset=False) - # this is similar to how MultiOutputClassifier handles predict_proba() if n_outputs > 1 - if isinstance(self.classifier_, MultiOutputClassifier): + if self.n_outputs == 1: + # Or self.classifier_.decision_function(X @ self.pxt_) + return X @ self.pxz_ + self.classifier_.intercept_ + else: return [ est_.decision_function(X @ self.pxt_) for est_ in self.classifier_.estimators_ ] - - # Or self.classifier_.decision_function(X @ self.pxt_) - return X @ self.pxz_ + self.classifier_.intercept_ else: T = check_array(T) - if isinstance(self.classifier_, MultiOutputClassifier): + if self.n_outputs == 1: + return T @ self.ptz_ + self.classifier_.intercept_ + else: return [ est_.decision_function(T) for est_ in self.classifier_.estimators_ ] - return T @ self.ptz_ + self.classifier_.intercept_ - def predict(self, X=None, T=None): """Predicts the property labels using classification on T.""" check_is_fitted(self, attributes=["pxz_", "ptz_"]) diff --git a/tests/test_pcovc.py b/tests/test_pcovc.py index 232953149..e43483d59 100644 --- a/tests/test_pcovc.py +++ b/tests/test_pcovc.py @@ -3,7 +3,8 @@ import numpy as np from sklearn import exceptions -from sklearn.datasets import load_breast_cancer as get_dataset +from sklearn.calibration import LinearSVC +from sklearn.datasets import load_iris as get_dataset from sklearn.decomposition import PCA from sklearn.linear_model import LogisticRegression, RidgeClassifier from sklearn.svm import LinearSVC @@ -98,7 +99,7 @@ def test_simple_prediction(self): Yp = pcovc.predict(self.X) self.assertLessEqual( - np.linalg.norm(Yp - Yhat) ** 2.0 / np.linalg.norm(Yp) ** 2.0, + np.linalg.norm(Yp - Yhat) ** 2.0 / np.linalg.norm(Yhat) ** 2.0, self.error_tol, ) @@ -580,15 +581,56 @@ def test_incompatible_coef_shape(self): class PCovCMultiOutputTest(PCovCBaseTest): - def test_projector_shapes(self): - pass + def test_prefit_multioutput(self): + """Check that PCovC works if a prefit classifier is passed when `n_ouputs > 1`.""" + classifier = MultiOutputClassifier(estimator=LogisticRegression()) + Y_double = np.column_stack((self.Y, self.Y)) - def test_decision_function(self): + classifier.fit(self.X, Y_double) + pcovc = self.model(mixing=0.25, classifier=classifier) + pcovc.fit(self.X, Y_double) + + W_classifier = np.hstack([est_.coef_.T for est_ in classifier.estimators_]) + Z_classifier = self.X @ W_classifier + + W_pcovc = np.hstack([est_.coef_.T for est_ in pcovc.z_classifier_.estimators_]) + Z_pcovc = self.X @ W_pcovc + + self.assertTrue(np.allclose(Z_classifier, Z_pcovc)) + self.assertTrue(np.allclose(W_classifier, W_pcovc)) + + def test_precomputed_multioutput(self): + """Check that PCovC works if classifier=`precomputed` and `n_ouputs > 1`.""" + classifier = MultiOutputClassifier(estimator=LogisticRegression()) + Y_double = np.column_stack((self.Y, self.Y)) + + classifier.fit(self.X, Y_double) + W = np.hstack([est_.coef_.T for est_ in classifier.estimators_]) + pcovc1 = self.model(mixing=0.5, classifier="precomputed", n_components=1) + pcovc1.fit(self.X, Y_double, W) + t1 = pcovc1.transform(self.X) + + pcovc2 = self.model(mixing=0.5, classifier=classifier, n_components=1) + pcovc2.fit(self.X, Y_double) + t2 = pcovc2.transform(self.X) + + self.assertTrue(np.linalg.norm(t1 - t2) < self.error_tol) + + # Now check for match when W is not passed: + pcovc3 = self.model(mixing=0.5, classifier="precomputed", n_components=1) + pcovc3.fit(self.X, Y_double) + t3 = pcovc3.transform(self.X) + + self.assertTrue(np.linalg.norm(t3 - t2) < self.error_tol) + self.assertTrue(np.linalg.norm(t3 - t1) < self.error_tol) + + def test_Z_shape_multioutput(self): + """Check that PCovC returns the evidence Z in the desired form when `n_ouputs > 1`.""" pcovc = PCovC( classifier=MultiOutputClassifier(LogisticRegression()), n_components=2 ) - Y_double = np.column_stack((self.Y, self.Y[::-1])) + Y_double = np.column_stack((self.Y, self.Y)) pcovc.fit(self.X, Y_double) Z = pcovc.decision_function(self.X) @@ -602,6 +644,20 @@ def test_decision_function(self): self.assertEqual(self.X.shape[0], z_slice.shape[0]) self.assertEqual(est.coef_.shape[0], z_slice.shape[1]) + def test_decision_function_multioutput(self): + """Check that PCovC's decision_function works in edge cases when `n_ouputs > 1`.""" + pcovc = self.model(classifier=MultiOutputClassifier(estimator=LinearSVC())) + pcovc.fit(self.X, np.column_stack((self.Y, self.Y))) + with self.assertRaises(ValueError) as cm: + _ = pcovc.decision_function() + self.assertEqual( + str(cm.exception), + "Either X or T must be supplied.", + ) + + T = pcovc.transform(self.X) + _ = pcovc.decision_function(T=T) + if __name__ == "__main__": unittest.main(verbosity=2) From d0dc35cc88cdbcc37053d1f06fd8b309fe659ff5 Mon Sep 17 00:00:00 2001 From: Rhushil Vasavada Date: Mon, 30 Jun 2025 11:09:12 -0500 Subject: [PATCH 08/27] Adding multioutput support for KPCovC --- src/skmatter/decomposition/_kernel_pcovc.py | 99 ++++++++++----- src/skmatter/decomposition/_pcovc.py | 127 ++++++-------------- src/skmatter/utils/_pcovc_utils.py | 6 +- tests/test_kernel_pcovc.py | 125 ++++++++++++++++--- tests/test_pcovc.py | 16 ++- 5 files changed, 231 insertions(+), 142 deletions(-) diff --git a/src/skmatter/decomposition/_kernel_pcovc.py b/src/skmatter/decomposition/_kernel_pcovc.py index e8965a223..2426573f7 100644 --- a/src/skmatter/decomposition/_kernel_pcovc.py +++ b/src/skmatter/decomposition/_kernel_pcovc.py @@ -1,6 +1,7 @@ import numpy as np from sklearn import clone +from sklearn.multioutput import MultiOutputClassifier from sklearn.svm import LinearSVC from sklearn.discriminant_analysis import LinearDiscriminantAnalysis from sklearn.linear_model import ( @@ -52,6 +53,9 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): n_components == n_samples + n_outputs : int + The number of outputs when ``fit`` is performed. + svd_solver : {'auto', 'full', 'arpack', 'randomized'}, default='auto' If auto : The solver is selected by a default policy based on `X.shape` and @@ -78,13 +82,14 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): - ``sklearn.linear_model.LogisticRegressionCV()`` - ``sklearn.svm.LinearSVC()`` - ``sklearn.discriminant_analysis.LinearDiscriminantAnalysis()`` + - ``sklearn.multioutput.MultiOutputClassifier()`` - ``sklearn.linear_model.RidgeClassifier()`` - ``sklearn.linear_model.RidgeClassifierCV()`` - ``sklearn.linear_model.Perceptron()`` If a pre-fitted classifier is provided, it is used to compute :math:`{\mathbf{Z}}`. - If None, ``sklearn.linear_model.LogisticRegression()`` - is used as the classifier. + If None and ``n_outputs < 2``, ``sklearn.linear_model.LogisticRegression()`` is used. + If None and ``n_outputs == 2``, ``sklearn.multioutput.MultiOutputClassifier()`` is used. kernel : {"linear", "poly", "rbf", "sigmoid", "precomputed"} or callable, default="linear" Kernel. @@ -132,6 +137,9 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): Attributes ---------- + n_outputs : int + The number of outputs when ``fit`` is performed. + classifier : estimator object The linear classifier passed for fitting. If pre-fitted, it is assummed to be fit on a precomputed kernel :math:`\mathbf{K}` and :math:`\mathbf{Y}`. @@ -268,9 +276,11 @@ def fit(self, X, Y, W=None): self: object Returns the instance itself. """ - X, Y = validate_data(self, X, Y, y_numeric=False) + X, Y = validate_data(self, X, Y, multi_output=True, y_numeric=False) + check_classification_targets(Y) self.classes_ = np.unique(Y) + self.n_outputs = 1 if Y.ndim == 1 else Y.shape[1] super()._set_fit_params(X) @@ -285,6 +295,7 @@ def fit(self, X, Y, W=None): LogisticRegressionCV, LinearSVC, LinearDiscriminantAnalysis, + MultiOutputClassifier, RidgeClassifier, RidgeClassifierCV, SGDClassifier, @@ -300,27 +311,37 @@ def fit(self, X, Y, W=None): ", or `precomputed`" ) - if self.classifier != "precomputed": - if self.classifier is None: - classifier = LogisticRegression() - else: - classifier = self.classifier + multioutput = self.n_outputs != 1 + precomputed = self.classifier == "precomputed" - # for convergence warnings - if hasattr(classifier, "max_iter") and ( - classifier.max_iter is None or classifier.max_iter < 500 - ): - classifier.max_iter = 500 + if self.classifier is None or precomputed: + # used as the default classifier for subsequent computations + classifier = ( + MultiOutputClassifier(LogisticRegression()) + if multioutput + else LogisticRegression() + ) + else: + classifier = self.classifier - # Check if classifier is fitted; if not, fit with precomputed K - self.z_classifier_ = check_cl_fit(classifier, K, Y) - W = self.z_classifier_.coef_.T + if hasattr(classifier, "max_iter") and ( + classifier.max_iter is None or classifier.max_iter < 500 + ): + classifier.max_iter = 500 + + if precomputed and W is None: + _ = clone(classifier).fit(K, Y) + if multioutput: + W = np.hstack([_.coef_.T for _ in _.estimators_]) + else: + W = _.coef_.T else: - # If precomputed, use default classifier to predict Y from T - classifier = LogisticRegression(max_iter=500) - if W is None: - W = LogisticRegression().fit(K, Y).coef_.T + self.z_classifier_ = check_cl_fit(classifier, K, Y) + if multioutput: + W = np.hstack([est_.coef_.T for est_ in self.z_classifier_.estimators_]) + else: + W = self.z_classifier_.coef_.T Z = K @ W @@ -333,10 +354,16 @@ def fit(self, X, Y, W=None): self.classifier_ = clone(classifier).fit(K @ self.pkt_, Y) - self.ptz_ = self.classifier_.coef_.T - self.pkz_ = self.pkt_ @ self.ptz_ + if multioutput: + self.ptz_ = np.hstack( + [est_.coef_.T for est_ in self.classifier_.estimators_] + ) + self.pkz_ = self.pkt_ @ self.ptz_ + else: + self.ptz_ = self.classifier_.coef_.T + self.pkz_ = self.pkt_ @ self.ptz_ - if len(Y.shape) == 1 and type_of_target(Y) == "binary": + if not multioutput and type_of_target(Y) == "binary": self.pkz_ = self.pkz_.reshape( K.shape[1], ) @@ -345,6 +372,7 @@ def fit(self, X, Y, W=None): ) self.components_ = self.pkt_.T # for sklearn compatibility + return self def predict(self, X=None, T=None): @@ -424,9 +452,12 @@ def decision_function(self, X=None, T=None): Returns ------- - Z : numpy.ndarray, shape (n_samples,) or (n_samples, n_classes) + Z : numpy.ndarray, shape (n_samples,) or (n_samples, n_classes), or a list of \ + n_outputs such arrays if n_outputs > 1 Confidence scores. For binary classification, has shape `(n_samples,)`, - for multiclass classification, has shape `(n_samples, n_classes)` + for multiclass classification, has shape `(n_samples, n_classes)`. + If n_outputs > 1, the list can contain arrays with differing shapes + depending on the number of classes in each output of Y. """ check_is_fitted(self, attributes=["pkz_", "ptz_"]) @@ -439,9 +470,21 @@ def decision_function(self, X=None, T=None): if self.center: K = self.centerer_.transform(K) - # Or self.classifier_.decision_function(K @ self.pkt_) - return K @ self.pkz_ + self.classifier_.intercept_ + if self.n_outputs == 1: + # Or self.classifier_.decision_function(K @ self.pkt_) + return K @ self.pkz_ + self.classifier_.intercept_ + else: + return [ + est_.decision_function(K @ self.pkt_) + for est_ in self.classifier_.estimators_ + ] else: T = check_array(T) - return T @ self.ptz_ + self.classifier_.intercept_ + + if self.n_outputs == 1: + T @ self.ptz_ + self.classifier_.intercept_ + else: + return [ + est_.decision_function(T) for est_ in self.classifier_.estimators_ + ] diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index e83f177ea..90cf25121 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -22,11 +22,11 @@ # No inheritance from MultiOutputMixin because decision_function would fail -# test_check_estimator.py 'check_classifier_multioutput' (line 2479 of estimator_checks.py) -# - this is the only test for MultiOutputClassifiers, so is it OK to exclude this tag? +# test_check_estimator.py 'check_classifier_multioutput' (line 2479 of estimator_checks.py). +# This is the only test for multioutput classifiers, so is it OK to exclude this tag? # did a search of all classifiers that inherit from MultiOutputMixin - none of them implement -# decision function, so I don't think we need to inherit +# decision function class PCovC(LinearClassifierMixin, _BasePCov): @@ -120,6 +120,7 @@ class PCovC(LinearClassifierMixin, _BasePCov): - ``sklearn.linear_model.LogisticRegressionCV()`` - ``sklearn.svm.LinearSVC()`` - ``sklearn.discriminant_analysis.LinearDiscriminantAnalysis()`` + - ``sklearn.multioutput.MultiOutputClassifier()`` - ``sklearn.linear_model.RidgeClassifier()`` - ``sklearn.linear_model.RidgeClassifierCV()`` - ``sklearn.linear_model.Perceptron()`` @@ -131,8 +132,8 @@ class PCovC(LinearClassifierMixin, _BasePCov): `sklearn.pipeline.Pipeline` with model caching. In such cases, the classifier will be re-fitted on the same training data as the composite estimator. - If None and ``Y.ndim < 2``, ``sklearn.linear_model.LogisticRegression()`` is used. - If None and ``Y.ndim == 2``, ``sklearn.multioutput.MultiOutputClassifier()`` is used. + If None and ``n_outputs < 2``, ``sklearn.linear_model.LogisticRegression()`` is used. + If None and ``n_outputs == 2``, ``sklearn.multioutput.MultiOutputClassifier()`` is used. iterated_power : int or 'auto', default='auto' Number of iterations for the power method computed by @@ -164,6 +165,9 @@ class PCovC(LinearClassifierMixin, _BasePCov): n_components, or the lesser value of n_features and n_samples if n_components is None. + n_outputs : int + The number of outputs when ``fit`` is performed. + classifier : estimator object The linear classifier passed for fitting. @@ -263,16 +267,14 @@ def fit(self, X, Y, W=None): Y : numpy.ndarray, shape (n_samples,) or (n_samples, n_outputs) Training data, where n_samples is the number of samples and - n_outputs is the number of outputs. If ``self.classifier`` is an instance - of ``sklearn.multioutput.MultiOutputClassifier()``, Y can be of shape - (n_samples, n_outputs). + n_outputs is the number of outputs. W : numpy.ndarray, shape (n_features, n_classes) Classification weights, optional when classifier is ``precomputed``. If not passed, it is assumed that the weights will be taken from a linear classifier fit between :math:`\mathbf{X}` and :math:`\mathbf{Y}`. - In the case of a multioutput classifier ``classifier``, - `` W = np.hstack([est_.coef_.T for est_ in classifier.estimators_])``. + In the multioutput case, + `` W = np.hstack([est_.coef_.T for est_ in classifier.estimators_])``. """ X, Y = validate_data(self, X, Y, multi_output=True, y_numeric=False) @@ -303,49 +305,31 @@ def fit(self, X, Y, W=None): ", or `precomputed`" ) - if self.n_outputs == 1 and isinstance(self.classifier, MultiOutputClassifier): - raise ValueError( - "Classifier cannot be an instance of `MultiOutputClassifier` when Y is 1D" - ) + multioutput = self.n_outputs != 1 + precomputed = self.classifier == "precomputed" - if ( - self.n_outputs != 1 - and self.classifier not in ["precomputed", None] - and not ( - isinstance(self.classifier, MultiOutputClassifier) - or self.classifier == "precomputed" - ) - ): - raise ValueError( - "Classifier must be an instance of `MultiOutputClassifier` when Y is 2D" + if self.classifier is None or precomputed: + # used as the default classifier for subsequent computations + classifier = ( + MultiOutputClassifier(LogisticRegression()) + if multioutput + else LogisticRegression() ) + else: + classifier = self.classifier - if self.n_outputs == 1: - if self.classifier != "precomputed": - classifier = self.classifier or LogisticRegression() - self.z_classifier_ = check_cl_fit(classifier, X, Y) - W = self.z_classifier_.coef_.T - + if precomputed and W is None: + _ = clone(classifier).fit(X, Y) + if multioutput: + W = np.hstack([_.coef_.T for _ in _.estimators_]) else: - # to be used later on as the classifier fit between T and Y - classifier = LogisticRegression() - if W is None: - W = clone(classifier).fit(X, Y).coef_.T - + W = _.coef_.T else: - if self.classifier != "precomputed": - classifier = self.classifier or MultiOutputClassifier( - estimator=LogisticRegression() - ) - self.z_classifier_ = check_cl_fit(classifier, X, Y) + self.z_classifier_ = check_cl_fit(classifier, X, Y) + if multioutput: W = np.hstack([est_.coef_.T for est_ in self.z_classifier_.estimators_]) - else: - # to be used later on as the classifier fit between T and Y - classifier = MultiOutputClassifier(estimator=LogisticRegression()) - if W is None: - _ = clone(classifier).fit(X, Y) - W = np.hstack([_.coef_.T for _ in _.estimators_]) + W = self.z_classifier_.coef_.T Z = X @ W @@ -358,11 +342,7 @@ def fit(self, X, Y, W=None): # classifier and steal weights to get pxz and ptz self.classifier_ = clone(classifier).fit(X @ self.pxt_, Y) - if self.n_outputs == 1: - self.ptz_ = self.classifier_.coef_.T - # print(self.ptz_.shape) - self.pxz_ = self.pxt_ @ self.ptz_ - else: + if multioutput: self.ptz_ = np.hstack( [est_.coef_.T for est_ in self.classifier_.estimators_] ) @@ -370,9 +350,13 @@ def fit(self, X, Y, W=None): # print(f"ptz {self.ptz_.shape}") self.pxz_ = self.pxt_ @ self.ptz_ # print(f"pxz {self.pxz_.shape}") + else: + self.ptz_ = self.classifier_.coef_.T + # print(self.ptz_.shape) + self.pxz_ = self.pxt_ @ self.ptz_ # print(self.ptz_.shape) - if len(Y.shape) == 1 and type_of_target(Y) == "binary": + if not multioutput and type_of_target(Y) == "binary": self.pxz_ = self.pxz_.reshape( X.shape[1], ) @@ -472,9 +456,9 @@ def decision_function(self, X=None, T=None): Z : numpy.ndarray, shape (n_samples,) or (n_samples, n_classes), or a list of \ n_outputs such arrays if n_outputs > 1 Confidence scores. For binary classification, has shape `(n_samples,)`, - for multiclass classification, has shape `(n_samples, n_classes)`. If n_outputs > 1, - the list can contain arrays with differing shapes depending on the - number of classes in each output of Y. + for multiclass classification, has shape `(n_samples, n_classes)`. + If n_outputs > 1, the list can contain arrays with differing shapes + depending on the number of classes in each output of Y. """ check_is_fitted(self, attributes=["pxz_", "ptz_"]) @@ -529,36 +513,3 @@ def transform(self, X=None): and n_features is the number of features. """ return super().transform(X) - - # def score(self, X, Y, sample_weight=None): - # """Return the accuracy on the given test data and labels. Contains support - # for multiclass-multioutput data. - - # Parameters - # ---------- - # X : array-like of shape (n_samples, n_features) - # Test samples. - - # Y : array-like of shape (n_samples,) or (n_samples, n_outputs) - # True labels for `X`. - - # sample_weight : array-like of shape (n_samples,), default=None - # Sample weights. Can only be used if the PCovC instance - # has been trained on single-target data. - - # Returns - # ------- - # score : float - # Accuracy scores. If the PCovC instance was trained on a 1D Y, - # this will call the ``score()`` function defined by - # ``sklearn.base.ClassifierMixin``. If trained on a 2D Y, this will - # call the ``score()`` function defined by - # ``sklearn.multioutput.MultiOutputClassifier``. - # """ - # X, Y = validate_data(self, X, Y, reset=False) - - # if isinstance(self.classifier_, MultiOutputClassifier): - # # LinearClassifierMixin.score fails with multioutput-multiclass Y - # return self.classifier_.score(X @ self.pxt_, Y) - # else: - # return self.classifier_.score(X @ self.pxt_, Y, sample_weight=sample_weight) diff --git a/src/skmatter/utils/_pcovc_utils.py b/src/skmatter/utils/_pcovc_utils.py index 3203dca82..e1f346b85 100644 --- a/src/skmatter/utils/_pcovc_utils.py +++ b/src/skmatter/utils/_pcovc_utils.py @@ -45,9 +45,9 @@ def check_cl_fit(classifier, X, y): # number of classes in y if isinstance(fitted_classifier, MultiOutputClassifier): for est_ in fitted_classifier.estimators_: - check_cl_coef(X, est_.coef_, len(est_.classes_)) + _check_cl_coef(X, est_.coef_, len(est_.classes_)) else: - check_cl_coef(X, fitted_classifier.coef_, len(np.unique(y))) + _check_cl_coef(X, fitted_classifier.coef_, len(np.unique(y))) except NotFittedError: fitted_classifier = clone(classifier) @@ -56,7 +56,7 @@ def check_cl_fit(classifier, X, y): return fitted_classifier -def check_cl_coef(X, classifier_coef_, n_classes): +def _check_cl_coef(X, classifier_coef_, n_classes): if n_classes == 2: if classifier_coef_.shape[0] != 1: raise ValueError( diff --git a/tests/test_kernel_pcovc.py b/tests/test_kernel_pcovc.py index 9b29b8437..0b4e2e385 100644 --- a/tests/test_kernel_pcovc.py +++ b/tests/test_kernel_pcovc.py @@ -4,10 +4,11 @@ from sklearn import exceptions from sklearn.calibration import LinearSVC from sklearn.datasets import load_breast_cancer as get_dataset +from sklearn.multioutput import MultiOutputClassifier from sklearn.naive_bayes import GaussianNB from sklearn.utils.validation import check_X_y from sklearn.preprocessing import StandardScaler -from sklearn.linear_model import LogisticRegression, RidgeClassifier +from sklearn.linear_model import LogisticRegression, Perceptron, RidgeClassifier from sklearn.metrics.pairwise import pairwise_kernels from skmatter.decomposition import KernelPCovC @@ -30,17 +31,12 @@ def __init__(self, *args, **kwargs): scaler = StandardScaler() self.X = scaler.fit_transform(self.X) - self.model = ( - lambda mixing=0.5, - classifier=LogisticRegression(), - n_components=4, - **kwargs: KernelPCovC( - mixing=mixing, - classifier=classifier, - n_components=n_components, - svd_solver=kwargs.pop("svd_solver", "full"), - **kwargs, - ) + self.model = lambda mixing=0.5, classifier=LogisticRegression(), n_components=4, **kwargs: KernelPCovC( + mixing=mixing, + classifier=classifier, + n_components=n_components, + svd_solver=kwargs.pop("svd_solver", "full"), + **kwargs, ) def setUp(self): @@ -217,7 +213,10 @@ def test_prefit_classifier(self): classifier = LinearSVC() classifier.fit(K, self.Y) - kpcovc = KernelPCovC(mixing=0.5, classifier=classifier, **kernel_params) + kpcovc = KernelPCovC( + mixing=0.5, + classifier=classifier, + ) kpcovc.fit(self.X, self.Y) Z_classifier = classifier.decision_function(K) @@ -256,7 +255,7 @@ def test_incompatible_classifier(self): str(cm.exception), "Classifier must be an instance of " "`LogisticRegression`, `LogisticRegressionCV`, `LinearSVC`, " - "`LinearDiscriminantAnalysis`, `RidgeClassifier`, " + "`LinearDiscriminantAnalysis`, `MultiOutputClassifier`, `RidgeClassifier`, " "`RidgeClassifierCV`, `SGDClassifier`, `Perceptron`, " "or `precomputed`", ) @@ -484,5 +483,103 @@ def test_bad_n_components(self): ) +class KernelPCovCMultiOutputTest(KernelPCovCBaseTest): + + def test_prefit_multioutput(self): + """Check that KPCovC works if a prefit classifier is passed when `n_outputs > 1`.""" + kernel_params = {"kernel": "sigmoid", "gamma": 1, "degree": 3, "coef0": 0} + K = pairwise_kernels( + self.X, metric="sigmoid", filter_params=True, **kernel_params + ) + + classifier = MultiOutputClassifier(estimator=LogisticRegression()) + Y_double = np.column_stack((self.Y, self.Y)) + + classifier.fit(K, Y_double) + kpcovc = self.model( + mixing=0.10, + classifier=classifier, + ) + kpcovc.fit(self.X, Y_double) + + W_classifier = np.hstack([est_.coef_.T for est_ in classifier.estimators_]) + Z_classifier = K @ W_classifier + + W_kpcovc = np.hstack( + [est_.coef_.T for est_ in kpcovc.z_classifier_.estimators_] + ) + Z_kpcovc = K @ W_kpcovc + + self.assertTrue(np.allclose(Z_classifier, Z_kpcovc)) + self.assertTrue(np.allclose(W_classifier, W_kpcovc)) + + def test_precomputed_multioutput(self): + """Check that KPCovC works if classifier=`precomputed` and `n_outputs > 1`.""" + kernel_params = {"kernel": "linear", "gamma": 5, "degree": 3, "coef0": 2} + K = pairwise_kernels( + self.X, metric="linear", filter_params=True, **kernel_params + ) + + classifier = MultiOutputClassifier(estimator=LogisticRegression()) + Y_double = np.column_stack((self.Y, self.Y)) + + classifier.fit(K, Y_double) + W = np.hstack([est_.coef_.T for est_ in classifier.estimators_]) + + kpcovc1 = self.model(mixing=0.5, classifier="precomputed", **kernel_params) + kpcovc1.fit(self.X, Y_double, W) + t1 = kpcovc1.transform(self.X) + + kpcovc2 = self.model(mixing=0.5, classifier=classifier, **kernel_params) + kpcovc2.fit(self.X, Y_double) + t2 = kpcovc2.transform(self.X) + + self.assertTrue(np.linalg.norm(t1 - t2) < self.error_tol) + + # Now check for match when W is not passed: + kpcovc3 = self.model(mixing=0.5, classifier="precomputed", **kernel_params) + kpcovc3.fit(self.X, Y_double) + t3 = kpcovc3.transform(self.X) + + self.assertTrue(np.linalg.norm(t3 - t2) < self.error_tol) + self.assertTrue(np.linalg.norm(t3 - t1) < self.error_tol) + + def test_Z_shape_multioutput(self): + """Check that KPCovC returns the evidence Z in the desired form when `n_outputs > 1`.""" + kpcovc = KernelPCovC(classifier=MultiOutputClassifier(estimator=Perceptron())) + + Y_double = np.column_stack((self.Y, self.Y)) + kpcovc.fit(self.X, Y_double) + + Z = kpcovc.decision_function(self.X) + + # list of (n_samples, ) arrays when each column of Y is binary + self.assertEqual(len(Z), Y_double.shape[1]) + + for est, z_slice in zip(kpcovc.z_classifier_.estimators_, Z): + with self.subTest(type="z_arrays"): + # each array is shape (n_samples, ): + self.assertEqual(self.X.shape[0], z_slice.shape[0]) + self.assertEqual(z_slice.ndim, 1) + + def test_decision_function_multioutput(self): + """Check that KPCovC's decision_function works in edge cases when `n_outputs > 1`.""" + kpcovc = self.model( + classifier=MultiOutputClassifier(estimator=LinearSVC()), center=True + ) + kpcovc.fit(self.X, np.column_stack((self.Y, self.Y))) + + with self.assertRaises(ValueError) as cm: + _ = kpcovc.decision_function() + self.assertEqual( + str(cm.exception), + "Either X or T must be supplied.", + ) + + _ = kpcovc.decision_function(self.X) + T = kpcovc.transform(self.X) + _ = kpcovc.decision_function(T=T) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/test_pcovc.py b/tests/test_pcovc.py index e43483d59..4f232137b 100644 --- a/tests/test_pcovc.py +++ b/tests/test_pcovc.py @@ -582,7 +582,7 @@ def test_incompatible_coef_shape(self): class PCovCMultiOutputTest(PCovCBaseTest): def test_prefit_multioutput(self): - """Check that PCovC works if a prefit classifier is passed when `n_ouputs > 1`.""" + """Check that PCovC works if a prefit classifier is passed when `n_outputs > 1`.""" classifier = MultiOutputClassifier(estimator=LogisticRegression()) Y_double = np.column_stack((self.Y, self.Y)) @@ -600,7 +600,7 @@ def test_prefit_multioutput(self): self.assertTrue(np.allclose(W_classifier, W_pcovc)) def test_precomputed_multioutput(self): - """Check that PCovC works if classifier=`precomputed` and `n_ouputs > 1`.""" + """Check that PCovC works if classifier=`precomputed` and `n_outputs > 1`.""" classifier = MultiOutputClassifier(estimator=LogisticRegression()) Y_double = np.column_stack((self.Y, self.Y)) @@ -625,27 +625,25 @@ def test_precomputed_multioutput(self): self.assertTrue(np.linalg.norm(t3 - t1) < self.error_tol) def test_Z_shape_multioutput(self): - """Check that PCovC returns the evidence Z in the desired form when `n_ouputs > 1`.""" - pcovc = PCovC( - classifier=MultiOutputClassifier(LogisticRegression()), n_components=2 - ) + """Check that PCovC returns the evidence Z in the desired form when `n_outputs > 1`.""" + pcovc = PCovC() Y_double = np.column_stack((self.Y, self.Y)) pcovc.fit(self.X, Y_double) Z = pcovc.decision_function(self.X) - # list of (n_samples, n_classes) arrays + # list of (n_samples, n_classes) arrays when each column of Y is multiclass self.assertEqual(len(Z), Y_double.shape[1]) for est, z_slice in zip(pcovc.z_classifier_.estimators_, Z): with self.subTest(type="z_arrays"): - # each array is shape (n_samples, n_classes) + # each array is shape (n_samples, n_classes): self.assertEqual(self.X.shape[0], z_slice.shape[0]) self.assertEqual(est.coef_.shape[0], z_slice.shape[1]) def test_decision_function_multioutput(self): - """Check that PCovC's decision_function works in edge cases when `n_ouputs > 1`.""" + """Check that PCovC's decision_function works in edge cases when `n_outputs > 1`.""" pcovc = self.model(classifier=MultiOutputClassifier(estimator=LinearSVC())) pcovc.fit(self.X, np.column_stack((self.Y, self.Y))) with self.assertRaises(ValueError) as cm: From aef9d917a5e9020b1a7ce365fb15c08d35ef8ffb Mon Sep 17 00:00:00 2001 From: Rhushil Vasavada Date: Tue, 26 Aug 2025 11:55:19 -0500 Subject: [PATCH 09/27] Fixes after rebase --- src/skmatter/decomposition/_pcovc.py | 10 +++++----- tests/test_kernel_pcovc.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index 90cf25121..25decb296 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -165,7 +165,7 @@ class PCovC(LinearClassifierMixin, _BasePCov): n_components, or the lesser value of n_features and n_samples if n_components is None. - n_outputs : int + n_outputs_ : int The number of outputs when ``fit`` is performed. classifier : estimator object @@ -280,7 +280,7 @@ def fit(self, X, Y, W=None): check_classification_targets(Y) self.classes_ = np.unique(Y) - self.n_outputs = 1 if Y.ndim == 1 else Y.shape[1] + self.n_outputs_ = 1 if Y.ndim == 1 else Y.shape[1] super()._set_fit_params(X) @@ -305,7 +305,7 @@ def fit(self, X, Y, W=None): ", or `precomputed`" ) - multioutput = self.n_outputs != 1 + multioutput = self.n_outputs_ != 1 precomputed = self.classifier == "precomputed" if self.classifier is None or precomputed: @@ -468,7 +468,7 @@ def decision_function(self, X=None, T=None): if X is not None: X = validate_data(self, X, reset=False) - if self.n_outputs == 1: + if self.n_outputs_ == 1: # Or self.classifier_.decision_function(X @ self.pxt_) return X @ self.pxz_ + self.classifier_.intercept_ else: @@ -479,7 +479,7 @@ def decision_function(self, X=None, T=None): else: T = check_array(T) - if self.n_outputs == 1: + if self.n_outputs_ == 1: return T @ self.ptz_ + self.classifier_.intercept_ else: return [ diff --git a/tests/test_kernel_pcovc.py b/tests/test_kernel_pcovc.py index 0b4e2e385..677d08183 100644 --- a/tests/test_kernel_pcovc.py +++ b/tests/test_kernel_pcovc.py @@ -556,7 +556,7 @@ def test_Z_shape_multioutput(self): # list of (n_samples, ) arrays when each column of Y is binary self.assertEqual(len(Z), Y_double.shape[1]) - for est, z_slice in zip(kpcovc.z_classifier_.estimators_, Z): + for z_slice in Z: with self.subTest(type="z_arrays"): # each array is shape (n_samples, ): self.assertEqual(self.X.shape[0], z_slice.shape[0]) From 02a9edf2912d7d5399a30ad7ebdb4961f3d9b645 Mon Sep 17 00:00:00 2001 From: Rhushil Vasavada Date: Thu, 4 Sep 2025 16:43:53 -0500 Subject: [PATCH 10/27] Remembering to add TODOs after last week conversation --- src/skmatter/decomposition/_kernel_pcovc.py | 2 ++ src/skmatter/decomposition/_pcovc.py | 2 ++ tests/test_kernel_pcovc.py | 2 ++ tests/test_pcovc.py | 2 +- 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/skmatter/decomposition/_kernel_pcovc.py b/src/skmatter/decomposition/_kernel_pcovc.py index 2426573f7..a8e538027 100644 --- a/src/skmatter/decomposition/_kernel_pcovc.py +++ b/src/skmatter/decomposition/_kernel_pcovc.py @@ -488,3 +488,5 @@ def decision_function(self, X=None, T=None): return [ est_.decision_function(T) for est_ in self.classifier_.estimators_ ] + +#TODO: add MultiOutputClassifier's score function for KPCovC to allow for multiclass-multioutput case \ No newline at end of file diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index 25decb296..df3a0ba59 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -513,3 +513,5 @@ def transform(self, X=None): and n_features is the number of features. """ return super().transform(X) + +#TODO: add MultiOutputClassifier's score function for PCovC to allow for multiclass-multioutput case \ No newline at end of file diff --git a/tests/test_kernel_pcovc.py b/tests/test_kernel_pcovc.py index 677d08183..63829a0f6 100644 --- a/tests/test_kernel_pcovc.py +++ b/tests/test_kernel_pcovc.py @@ -580,6 +580,8 @@ def test_decision_function_multioutput(self): T = kpcovc.transform(self.X) _ = kpcovc.decision_function(T=T) + #TODO: Add tests for addition of score function to pcovc.py + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/test_pcovc.py b/tests/test_pcovc.py index 4f232137b..22b782d91 100644 --- a/tests/test_pcovc.py +++ b/tests/test_pcovc.py @@ -656,6 +656,6 @@ def test_decision_function_multioutput(self): T = pcovc.transform(self.X) _ = pcovc.decision_function(T=T) - + #TODO: Add tests for addition of score function to pcovc.py if __name__ == "__main__": unittest.main(verbosity=2) From e008a653dee3b5dc0f490cefdda0d94951a27b28 Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Thu, 11 Sep 2025 15:22:20 -0500 Subject: [PATCH 11/27] Adding example --- examples/pcovc/PCovC_multioutput.py | 136 ++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 examples/pcovc/PCovC_multioutput.py diff --git a/examples/pcovc/PCovC_multioutput.py b/examples/pcovc/PCovC_multioutput.py new file mode 100644 index 000000000..42bc3bfc2 --- /dev/null +++ b/examples/pcovc/PCovC_multioutput.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# coding: utf-8 + +""" +Multioutput PCovC +================= +""" +# %% +# + +import numpy as np +import matplotlib.pyplot as plt + +from sklearn.datasets import load_digits +from sklearn.preprocessing import StandardScaler +from sklearn.decomposition import PCA +from sklearn.linear_model import LogisticRegressionCV +from sklearn.multioutput import MultiOutputClassifier + +from skmatter.decomposition import PCovC + +plt.rcParams["image.cmap"] = "tab10" +plt.rcParams["scatter.edgecolors"] = "k" +# %% +# +# +X, y = load_digits(return_X_y=True) +x_scaler = StandardScaler() +X_scaled = StandardScaler().fit_transform(X) + +np.unique(y) +# %% +# Let's begin by trying to make a PCovC map to separate the digits. +# This is a one-label, ten-class classification problem. +pca = PCA(n_components=2) +T_pca = pca.fit_transform(X_scaled, y) + +pcovc = PCovC(n_components=2, mixing=0.5) +T_pcovc = pcovc.fit_transform(X_scaled, y) + +fig, axs = plt.subplots(1, 2, figsize=(10, 6)) + +scat_pca = axs[0].scatter(T_pca[:, 0], T_pca[:, 1], c=y) +scat_pcovc = axs[1].scatter(T_pcovc[:, 0], T_pcovc[:, 1], c=y) +fig.colorbar(scat_pca, ax=axs, orientation="horizontal") + +# %% +# Next, let's try a two-label classification problem, with both labels +# being binary classification tasks. + +is_even = (y % 2).reshape(-1, 1) +is_less_than_five = (y < 5).reshape(-1, 1) + +y2 = np.hstack([is_even, is_less_than_five]) +y2.shape +# %% +# Here, we can build a map that considers both of these labels simultaneously. + +clf = MultiOutputClassifier(estimator=LogisticRegressionCV()) +pcovc = PCovC(n_components=2, mixing=0.5, classifier=clf) + +T_pcovc = pcovc.fit_transform(X_scaled, y2) + +fig, axs = plt.subplots(2, 3, figsize=(15, 10)) +cmap1 = "Set1" +cmap2 = "Set2" +cmap3 = "tab10" + +labels_list = [["Even", "Odd"], [">= 5", "< 5"]] + +for i, c, cmap in zip(range(3), [is_even, is_less_than_five, y], [cmap1, cmap2, cmap3]): + + scat_pca = axs[0, i].scatter(T_pca[:, 0], T_pca[:, 1], c=c, cmap=cmap) + axs[1, i].scatter(T_pcovc[:, 0], T_pcovc[:, 1], c=c, cmap=cmap) + + if i == 0 or i == 1: + handles, _ = scat_pca.legend_elements() + labels = labels_list[i] + axs[0, i].legend(handles, labels) + print(labels) + print(i) + print(handles) + + +axs[0, 0].set_title("Even/Odd") +axs[0, 1].set_title("Greater/Less than 5") +axs[0, 2].set_title("Digit") + +axs[0, 0].set_ylabel("PCA") +axs[1, 0].set_ylabel("PCovC") +fig.colorbar(scat_pca, ax=axs, orientation="horizontal") +# %% +# Let's try a more complicated example: + +num_holes = np.array( + [0 if i in [1, 2, 3, 5, 7] else 1 if i in [0, 4, 6, 9] else 2 for i in y] +).reshape(-1, 1) + +y3 = np.hstack([is_even, num_holes]) +# %% +# Now, we have a two-label classification +# problem, with one binary label and one label with three +# possible classes +clf = MultiOutputClassifier(estimator=LogisticRegressionCV()) +pcovc = PCovC(n_components=2, mixing=0.5, classifier=clf) + +T_pcovc = pcovc.fit_transform(X_scaled, y3) + +fig, axs = plt.subplots(2, 3, figsize=(15, 10)) +cmap1 = "Set1" +cmap2 = "Set3" +cmap3 = "tab10" + +labels_list = [["Even", "Odd"], ["0", "1", "2"]] + +for i, c, cmap in zip(range(3), [is_even, num_holes, y], [cmap1, cmap2, cmap3]): + + scat_pca = axs[0, i].scatter(T_pca[:, 0], T_pca[:, 1], c=c, cmap=cmap) + axs[1, i].scatter(T_pcovc[:, 0], T_pcovc[:, 1], c=c, cmap=cmap) + + if i == 0 or i == 1: + handles, _ = scat_pca.legend_elements() + labels = labels_list[i] + axs[0, i].legend(handles, labels) + print(labels) + print(i) + print(handles) + + +axs[0, 0].set_title("Even/Odd") +axs[0, 1].set_title("Number of Holes") +axs[0, 2].set_title("Digit") + +axs[0, 0].set_ylabel("PCA") +axs[1, 0].set_ylabel("PCovC") +fig.colorbar(scat_pca, ax=axs, orientation="horizontal") From 99571804a3f8906d66d5d1a979efb078aa6882fe Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Fri, 12 Sep 2025 15:12:13 -0500 Subject: [PATCH 12/27] Adding `score` methods for multiclass-multilabel problems --- src/skmatter/decomposition/_kernel_pcovc.py | 33 ++++++++++++++------- src/skmatter/decomposition/_pcovc.py | 13 +++++++- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/skmatter/decomposition/_kernel_pcovc.py b/src/skmatter/decomposition/_kernel_pcovc.py index a8e538027..16bcdecd2 100644 --- a/src/skmatter/decomposition/_kernel_pcovc.py +++ b/src/skmatter/decomposition/_kernel_pcovc.py @@ -53,7 +53,7 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): n_components == n_samples - n_outputs : int + n_outputs_ : int The number of outputs when ``fit`` is performed. svd_solver : {'auto', 'full', 'arpack', 'randomized'}, default='auto' @@ -88,8 +88,8 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): - ``sklearn.linear_model.Perceptron()`` If a pre-fitted classifier is provided, it is used to compute :math:`{\mathbf{Z}}`. - If None and ``n_outputs < 2``, ``sklearn.linear_model.LogisticRegression()`` is used. - If None and ``n_outputs == 2``, ``sklearn.multioutput.MultiOutputClassifier()`` is used. + If None and ``n_outputs_ < 2``, ``sklearn.linear_model.LogisticRegression()`` is used. + If None and ``n_outputs_ == 2``, ``sklearn.multioutput.MultiOutputClassifier()`` is used. kernel : {"linear", "poly", "rbf", "sigmoid", "precomputed"} or callable, default="linear" Kernel. @@ -137,7 +137,7 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): Attributes ---------- - n_outputs : int + n_outputs_ : int The number of outputs when ``fit`` is performed. classifier : estimator object @@ -280,7 +280,7 @@ def fit(self, X, Y, W=None): check_classification_targets(Y) self.classes_ = np.unique(Y) - self.n_outputs = 1 if Y.ndim == 1 else Y.shape[1] + self.n_outputs_ = 1 if Y.ndim == 1 else Y.shape[1] super()._set_fit_params(X) @@ -311,7 +311,7 @@ def fit(self, X, Y, W=None): ", or `precomputed`" ) - multioutput = self.n_outputs != 1 + multioutput = self.n_outputs_ != 1 precomputed = self.classifier == "precomputed" if self.classifier is None or precomputed: @@ -453,10 +453,10 @@ def decision_function(self, X=None, T=None): Returns ------- Z : numpy.ndarray, shape (n_samples,) or (n_samples, n_classes), or a list of \ - n_outputs such arrays if n_outputs > 1 + n_outputs_ such arrays if n_outputs_ > 1 Confidence scores. For binary classification, has shape `(n_samples,)`, for multiclass classification, has shape `(n_samples, n_classes)`. - If n_outputs > 1, the list can contain arrays with differing shapes + If n_outputs_ > 1, the list can contain arrays with differing shapes depending on the number of classes in each output of Y. """ check_is_fitted(self, attributes=["pkz_", "ptz_"]) @@ -470,7 +470,7 @@ def decision_function(self, X=None, T=None): if self.center: K = self.centerer_.transform(K) - if self.n_outputs == 1: + if self.n_outputs_ == 1: # Or self.classifier_.decision_function(K @ self.pkt_) return K @ self.pkz_ + self.classifier_.intercept_ else: @@ -482,11 +482,22 @@ def decision_function(self, X=None, T=None): else: T = check_array(T) - if self.n_outputs == 1: + if self.n_outputs_ == 1: T @ self.ptz_ + self.classifier_.intercept_ else: return [ est_.decision_function(T) for est_ in self.classifier_.estimators_ ] -#TODO: add MultiOutputClassifier's score function for KPCovC to allow for multiclass-multioutput case \ No newline at end of file + def score(self, X, y): + + # accuracy_score will handle everything but multiclass-multilabel + if self.n_outputs_ > 1 and len(self.classes_) > 2: + y_pred = self.predict(X) + return np.mean(np.all(y == y_pred, axis=1)) + + else: + return super().score(X, y) + + # Inherit the docstring from scikit-learn + score.__doc__ = LinearClassifierMixin.score.__doc__ diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index df3a0ba59..f86013f0c 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -514,4 +514,15 @@ def transform(self, X=None): """ return super().transform(X) -#TODO: add MultiOutputClassifier's score function for PCovC to allow for multiclass-multioutput case \ No newline at end of file + def score(self, X, y): + + # accuracy_score will handle everything but multiclass-multilabel + if self.n_outputs_ > 1 and len(self.classes_) > 2: + y_pred = self.predict(X) + return np.mean(np.all(y == y_pred, axis=1)) + + else: + return super().score(X, y) + + # Inherit the docstring from scikit-learn + score.__doc__ = LinearClassifierMixin.score.__doc__ From 92768604cbef1ec44f3dca6dd4abee4f03984128 Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Fri, 12 Sep 2025 15:23:43 -0500 Subject: [PATCH 13/27] Fix typo --- examples/pcovc/PCovC_multioutput.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pcovc/PCovC_multioutput.py b/examples/pcovc/PCovC_multioutput.py index 42bc3bfc2..863522ff3 100644 --- a/examples/pcovc/PCovC_multioutput.py +++ b/examples/pcovc/PCovC_multioutput.py @@ -100,7 +100,7 @@ # %% # Now, we have a two-label classification # problem, with one binary label and one label with three -# possible classes +# possible classes. clf = MultiOutputClassifier(estimator=LogisticRegressionCV()) pcovc = PCovC(n_components=2, mixing=0.5, classifier=clf) From 31abefbbe028053aa78337da1834b9fad72dd38e Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Fri, 12 Sep 2025 15:59:08 -0500 Subject: [PATCH 14/27] Making linter happy --- examples/pcovc/PCovC_multioutput.py | 2 -- src/skmatter/decomposition/_kernel_pcovc.py | 23 +++++++++----- src/skmatter/decomposition/_pcovc.py | 30 ++++++++---------- tests/test_kernel_pcovc.py | 34 +++++++++++++-------- tests/test_pcovc.py | 18 +++++++---- 5 files changed, 61 insertions(+), 46 deletions(-) diff --git a/examples/pcovc/PCovC_multioutput.py b/examples/pcovc/PCovC_multioutput.py index 863522ff3..f659b3015 100644 --- a/examples/pcovc/PCovC_multioutput.py +++ b/examples/pcovc/PCovC_multioutput.py @@ -69,7 +69,6 @@ labels_list = [["Even", "Odd"], [">= 5", "< 5"]] for i, c, cmap in zip(range(3), [is_even, is_less_than_five, y], [cmap1, cmap2, cmap3]): - scat_pca = axs[0, i].scatter(T_pca[:, 0], T_pca[:, 1], c=c, cmap=cmap) axs[1, i].scatter(T_pcovc[:, 0], T_pcovc[:, 1], c=c, cmap=cmap) @@ -114,7 +113,6 @@ labels_list = [["Even", "Odd"], ["0", "1", "2"]] for i, c, cmap in zip(range(3), [is_even, num_holes, y], [cmap1, cmap2, cmap3]): - scat_pca = axs[0, i].scatter(T_pca[:, 0], T_pca[:, 1], c=c, cmap=cmap) axs[1, i].scatter(T_pcovc[:, 0], T_pcovc[:, 1], c=c, cmap=cmap) diff --git a/src/skmatter/decomposition/_kernel_pcovc.py b/src/skmatter/decomposition/_kernel_pcovc.py index 16bcdecd2..7a23cb132 100644 --- a/src/skmatter/decomposition/_kernel_pcovc.py +++ b/src/skmatter/decomposition/_kernel_pcovc.py @@ -87,9 +87,17 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): - ``sklearn.linear_model.RidgeClassifierCV()`` - ``sklearn.linear_model.Perceptron()`` - If a pre-fitted classifier is provided, it is used to compute :math:`{\mathbf{Z}}`. - If None and ``n_outputs_ < 2``, ``sklearn.linear_model.LogisticRegression()`` is used. - If None and ``n_outputs_ == 2``, ``sklearn.multioutput.MultiOutputClassifier()`` is used. + If a pre-fitted classifier + is provided, it is used to compute :math:`{\mathbf{Z}}`. + Note that any pre-fitting of the classifier will be lost if `KernelPCovC` is + within a composite estimator that enforces cloning, e.g., + `sklearn.pipeline.Pipeline` with model caching. + In such cases, the classifier will be re-fitted on the same + training data as the composite estimator. + If None and ``n_outputs < 2``, ``sklearn.linear_model.LogisticRegression()`` is used. + If None and ``n_outputs >= 2``, a ``sklearn.multioutput.MultiOutputClassifier()`` is + constructed, with ``sklearn.linear_model.LogisticRegression()`` models used for each + label. kernel : {"linear", "poly", "rbf", "sigmoid", "precomputed"} or callable, default="linear" Kernel. @@ -455,8 +463,8 @@ def decision_function(self, X=None, T=None): Z : numpy.ndarray, shape (n_samples,) or (n_samples, n_classes), or a list of \ n_outputs_ such arrays if n_outputs_ > 1 Confidence scores. For binary classification, has shape `(n_samples,)`, - for multiclass classification, has shape `(n_samples, n_classes)`. - If n_outputs_ > 1, the list can contain arrays with differing shapes + for multiclass classification, has shape `(n_samples, n_classes)`. + If n_outputs_ > 1, the list can contain arrays with differing shapes depending on the number of classes in each output of Y. """ check_is_fitted(self, attributes=["pkz_", "ptz_"]) @@ -489,15 +497,14 @@ def decision_function(self, X=None, T=None): est_.decision_function(T) for est_ in self.classifier_.estimators_ ] - def score(self, X, y): - + def score(self, X, y, sample_weight=None): # accuracy_score will handle everything but multiclass-multilabel if self.n_outputs_ > 1 and len(self.classes_) > 2: y_pred = self.predict(X) return np.mean(np.all(y == y_pred, axis=1)) else: - return super().score(X, y) + return super().score(X, y, sample_weight) # Inherit the docstring from scikit-learn score.__doc__ = LinearClassifierMixin.score.__doc__ diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index f86013f0c..a400f7623 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -21,14 +21,6 @@ from skmatter.utils import check_cl_fit -# No inheritance from MultiOutputMixin because decision_function would fail -# test_check_estimator.py 'check_classifier_multioutput' (line 2479 of estimator_checks.py). -# This is the only test for multioutput classifiers, so is it OK to exclude this tag? - -# did a search of all classifiers that inherit from MultiOutputMixin - none of them implement -# decision function - - class PCovC(LinearClassifierMixin, _BasePCov): r"""Principal Covariates Classification (PCovC). @@ -133,7 +125,9 @@ class PCovC(LinearClassifierMixin, _BasePCov): In such cases, the classifier will be re-fitted on the same training data as the composite estimator. If None and ``n_outputs < 2``, ``sklearn.linear_model.LogisticRegression()`` is used. - If None and ``n_outputs == 2``, ``sklearn.multioutput.MultiOutputClassifier()`` is used. + If None and ``n_outputs >= 2``, a ``sklearn.multioutput.MultiOutputClassifier()`` is + constructed, with ``sklearn.linear_model.LogisticRegression()`` models used for each + label. iterated_power : int or 'auto', default='auto' Number of iterations for the power method computed by @@ -453,12 +447,13 @@ def decision_function(self, X=None, T=None): Returns ------- - Z : numpy.ndarray, shape (n_samples,) or (n_samples, n_classes), or a list of \ - n_outputs such arrays if n_outputs > 1 - Confidence scores. For binary classification, has shape `(n_samples,)`, - for multiclass classification, has shape `(n_samples, n_classes)`. - If n_outputs > 1, the list can contain arrays with differing shapes - depending on the number of classes in each output of Y. + Z : numpy.ndarray, shape (n_samples,) or (n_samples, n_classes), or + a list of n_outputs such arrays if n_outputs > 1. + Confidence scores. For binary classification, has shape + `(n_samples,)`, for multiclass classification, has shape + `(n_samples, n_classes)`. If n_outputs > 1, the list can + contain arrays with differing shapes depending on the number + of classes in each output of Y. """ check_is_fitted(self, attributes=["pxz_", "ptz_"]) @@ -514,15 +509,14 @@ def transform(self, X=None): """ return super().transform(X) - def score(self, X, y): - + def score(self, X, y, sample_weight=None): # accuracy_score will handle everything but multiclass-multilabel if self.n_outputs_ > 1 and len(self.classes_) > 2: y_pred = self.predict(X) return np.mean(np.all(y == y_pred, axis=1)) else: - return super().score(X, y) + return super().score(X, y, sample_weight) # Inherit the docstring from scikit-learn score.__doc__ = LinearClassifierMixin.score.__doc__ diff --git a/tests/test_kernel_pcovc.py b/tests/test_kernel_pcovc.py index 63829a0f6..725e25deb 100644 --- a/tests/test_kernel_pcovc.py +++ b/tests/test_kernel_pcovc.py @@ -2,7 +2,7 @@ import numpy as np from sklearn import exceptions -from sklearn.calibration import LinearSVC +from sklearn.svm import LinearSVC from sklearn.datasets import load_breast_cancer as get_dataset from sklearn.multioutput import MultiOutputClassifier from sklearn.naive_bayes import GaussianNB @@ -31,12 +31,17 @@ def __init__(self, *args, **kwargs): scaler = StandardScaler() self.X = scaler.fit_transform(self.X) - self.model = lambda mixing=0.5, classifier=LogisticRegression(), n_components=4, **kwargs: KernelPCovC( - mixing=mixing, - classifier=classifier, - n_components=n_components, - svd_solver=kwargs.pop("svd_solver", "full"), - **kwargs, + self.model = ( + lambda mixing=0.5, + classifier=LogisticRegression(), + n_components=4, + **kwargs: KernelPCovC( + mixing=mixing, + classifier=classifier, + n_components=n_components, + svd_solver=kwargs.pop("svd_solver", "full"), + **kwargs, + ) ) def setUp(self): @@ -484,9 +489,10 @@ def test_bad_n_components(self): class KernelPCovCMultiOutputTest(KernelPCovCBaseTest): - def test_prefit_multioutput(self): - """Check that KPCovC works if a prefit classifier is passed when `n_outputs > 1`.""" + """Check that KPCovC works if a prefit classifier + is passed when `n_outputs > 1`. + """ kernel_params = {"kernel": "sigmoid", "gamma": 1, "degree": 3, "coef0": 0} K = pairwise_kernels( self.X, metric="sigmoid", filter_params=True, **kernel_params @@ -545,7 +551,9 @@ def test_precomputed_multioutput(self): self.assertTrue(np.linalg.norm(t3 - t1) < self.error_tol) def test_Z_shape_multioutput(self): - """Check that KPCovC returns the evidence Z in the desired form when `n_outputs > 1`.""" + """Check that KPCovC returns the evidence Z in + the desired form when `n_outputs > 1`. + """ kpcovc = KernelPCovC(classifier=MultiOutputClassifier(estimator=Perceptron())) Y_double = np.column_stack((self.Y, self.Y)) @@ -563,7 +571,9 @@ def test_Z_shape_multioutput(self): self.assertEqual(z_slice.ndim, 1) def test_decision_function_multioutput(self): - """Check that KPCovC's decision_function works in edge cases when `n_outputs > 1`.""" + """Check that KPCovC's decision_function works + in edge cases when `n_outputs > 1`. + """ kpcovc = self.model( classifier=MultiOutputClassifier(estimator=LinearSVC()), center=True ) @@ -580,7 +590,7 @@ def test_decision_function_multioutput(self): T = kpcovc.transform(self.X) _ = kpcovc.decision_function(T=T) - #TODO: Add tests for addition of score function to pcovc.py + # TODO: Add tests for addition of score function to pcovc.py if __name__ == "__main__": diff --git a/tests/test_pcovc.py b/tests/test_pcovc.py index 22b782d91..f5e0ba829 100644 --- a/tests/test_pcovc.py +++ b/tests/test_pcovc.py @@ -3,7 +3,6 @@ import numpy as np from sklearn import exceptions -from sklearn.calibration import LinearSVC from sklearn.datasets import load_iris as get_dataset from sklearn.decomposition import PCA from sklearn.linear_model import LogisticRegression, RidgeClassifier @@ -580,9 +579,10 @@ def test_incompatible_coef_shape(self): class PCovCMultiOutputTest(PCovCBaseTest): - def test_prefit_multioutput(self): - """Check that PCovC works if a prefit classifier is passed when `n_outputs > 1`.""" + """Check that PCovC works if a prefit classifier + is passed when `n_outputs > 1`. + """ classifier = MultiOutputClassifier(estimator=LogisticRegression()) Y_double = np.column_stack((self.Y, self.Y)) @@ -625,7 +625,9 @@ def test_precomputed_multioutput(self): self.assertTrue(np.linalg.norm(t3 - t1) < self.error_tol) def test_Z_shape_multioutput(self): - """Check that PCovC returns the evidence Z in the desired form when `n_outputs > 1`.""" + """Check that PCovC returns the evidence Z in the + desired form when `n_outputs > 1`. + """ pcovc = PCovC() Y_double = np.column_stack((self.Y, self.Y)) @@ -643,7 +645,9 @@ def test_Z_shape_multioutput(self): self.assertEqual(est.coef_.shape[0], z_slice.shape[1]) def test_decision_function_multioutput(self): - """Check that PCovC's decision_function works in edge cases when `n_outputs > 1`.""" + """Check that PCovC's decision_function works in edge + cases when `n_outputs_ > 1`. + """ pcovc = self.model(classifier=MultiOutputClassifier(estimator=LinearSVC())) pcovc.fit(self.X, np.column_stack((self.Y, self.Y))) with self.assertRaises(ValueError) as cm: @@ -656,6 +660,8 @@ def test_decision_function_multioutput(self): T = pcovc.transform(self.X) _ = pcovc.decision_function(T=T) - #TODO: Add tests for addition of score function to pcovc.py + # TODO: Add tests for addition of score function to pcovc.py + + if __name__ == "__main__": unittest.main(verbosity=2) From d0e1adc7552805db7c63f1d97cdaebc9cc215b55 Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Fri, 12 Sep 2025 16:07:18 -0500 Subject: [PATCH 15/27] Example touch-ups --- examples/pcovc/PCovC_multioutput.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/examples/pcovc/PCovC_multioutput.py b/examples/pcovc/PCovC_multioutput.py index f659b3015..e8e2ae19f 100644 --- a/examples/pcovc/PCovC_multioutput.py +++ b/examples/pcovc/PCovC_multioutput.py @@ -43,6 +43,7 @@ scat_pca = axs[0].scatter(T_pca[:, 0], T_pca[:, 1], c=y) scat_pcovc = axs[1].scatter(T_pcovc[:, 0], T_pcovc[:, 1], c=y) fig.colorbar(scat_pca, ax=axs, orientation="horizontal") +fig.suptitle("Multiclass PCovC with One Label") # %% # Next, let's try a two-label classification problem, with both labels @@ -76,11 +77,7 @@ handles, _ = scat_pca.legend_elements() labels = labels_list[i] axs[0, i].legend(handles, labels) - print(labels) - print(i) - print(handles) - - + axs[0, 0].set_title("Even/Odd") axs[0, 1].set_title("Greater/Less than 5") axs[0, 2].set_title("Digit") @@ -88,6 +85,7 @@ axs[0, 0].set_ylabel("PCA") axs[1, 0].set_ylabel("PCovC") fig.colorbar(scat_pca, ax=axs, orientation="horizontal") +fig.suptitle("Multilabel PCovC with Binary Labels") # %% # Let's try a more complicated example: @@ -120,10 +118,6 @@ handles, _ = scat_pca.legend_elements() labels = labels_list[i] axs[0, i].legend(handles, labels) - print(labels) - print(i) - print(handles) - axs[0, 0].set_title("Even/Odd") axs[0, 1].set_title("Number of Holes") @@ -132,3 +126,4 @@ axs[0, 0].set_ylabel("PCA") axs[1, 0].set_ylabel("PCovC") fig.colorbar(scat_pca, ax=axs, orientation="horizontal") +fig.suptitle("Multiclass-Multilabel PCovC") \ No newline at end of file From c308bf2f11c825798782339fd8440818fe35bda4 Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Fri, 12 Sep 2025 16:08:06 -0500 Subject: [PATCH 16/27] Linting again --- examples/pcovc/PCovC_multioutput.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/pcovc/PCovC_multioutput.py b/examples/pcovc/PCovC_multioutput.py index e8e2ae19f..ddf08a4b0 100644 --- a/examples/pcovc/PCovC_multioutput.py +++ b/examples/pcovc/PCovC_multioutput.py @@ -77,7 +77,7 @@ handles, _ = scat_pca.legend_elements() labels = labels_list[i] axs[0, i].legend(handles, labels) - + axs[0, 0].set_title("Even/Odd") axs[0, 1].set_title("Greater/Less than 5") axs[0, 2].set_title("Digit") @@ -126,4 +126,4 @@ axs[0, 0].set_ylabel("PCA") axs[1, 0].set_ylabel("PCovC") fig.colorbar(scat_pca, ax=axs, orientation="horizontal") -fig.suptitle("Multiclass-Multilabel PCovC") \ No newline at end of file +fig.suptitle("Multiclass-Multilabel PCovC") From c20262cc29119147af5529dc7d796f034f72c248 Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Mon, 15 Sep 2025 11:32:14 -0500 Subject: [PATCH 17/27] New tests --- src/skmatter/decomposition/_kernel_pcovc.py | 31 +++++++++++------ src/skmatter/decomposition/_pcovc.py | 31 +++++++++++------ tests/test_kernel_pcovc.py | 34 +++++++++++++++--- tests/test_pcovc.py | 38 ++++++++++++++++++--- 4 files changed, 105 insertions(+), 29 deletions(-) diff --git a/src/skmatter/decomposition/_kernel_pcovc.py b/src/skmatter/decomposition/_kernel_pcovc.py index 7a23cb132..913ea9132 100644 --- a/src/skmatter/decomposition/_kernel_pcovc.py +++ b/src/skmatter/decomposition/_kernel_pcovc.py @@ -298,26 +298,37 @@ def fit(self, X, Y, W=None): self.centerer_ = KernelNormalizer() K = self.centerer_.fit_transform(K) - compatible_classifiers = ( + compatible_clfs = ( LogisticRegression, LogisticRegressionCV, LinearSVC, LinearDiscriminantAnalysis, - MultiOutputClassifier, RidgeClassifier, RidgeClassifierCV, SGDClassifier, Perceptron, + MultiOutputClassifier, ) - if self.classifier not in ["precomputed", None] and not isinstance( - self.classifier, compatible_classifiers - ): - raise ValueError( - "Classifier must be an instance of `" - f"{'`, `'.join(c.__name__ for c in compatible_classifiers)}`" - ", or `precomputed`" - ) + if self.classifier not in ["precomputed", None]: + if not isinstance(self.classifier, compatible_clfs): + raise ValueError( + "Classifier must be an instance of `" + f"{'`, `'.join(c.__name__ for c in compatible_clfs)}`" + ", or `precomputed`." + ) + + if isinstance(self.classifier, MultiOutputClassifier): + if not isinstance(self.classifier.estimator, compatible_clfs): + name = type(self.classifier.estimator).__name__ + raise ValueError( + "The instance of MultiOutputClassifier passed as the " + f"KernelPCovC classifier contains `{name}`, " + "which is not supported. The MultiOutputClassifier " + "must contain an instance of `" + f"{'`, `'.join(c.__name__ for c in compatible_clfs[:-1])}" + "`, or `precomputed`." + ) multioutput = self.n_outputs_ != 1 precomputed = self.classifier == "precomputed" diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index a400f7623..8e618e55e 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -278,26 +278,37 @@ def fit(self, X, Y, W=None): super()._set_fit_params(X) - compatible_classifiers = ( + compatible_clfs = ( LogisticRegression, LogisticRegressionCV, LinearSVC, LinearDiscriminantAnalysis, - MultiOutputClassifier, RidgeClassifier, RidgeClassifierCV, SGDClassifier, Perceptron, + MultiOutputClassifier, ) - if self.classifier not in ["precomputed", None] and not isinstance( - self.classifier, compatible_classifiers - ): - raise ValueError( - "Classifier must be an instance of `" - f"{'`, `'.join(c.__name__ for c in compatible_classifiers)}`" - ", or `precomputed`" - ) + if self.classifier not in ["precomputed", None]: + if not isinstance(self.classifier, compatible_clfs): + raise ValueError( + "Classifier must be an instance of `" + f"{'`, `'.join(c.__name__ for c in compatible_clfs)}`" + ", or `precomputed`." + ) + + if isinstance(self.classifier, MultiOutputClassifier): + if not isinstance(self.classifier.estimator, compatible_clfs): + name = type(self.classifier.estimator).__name__ + raise ValueError( + "The instance of MultiOutputClassifier passed as the " + f"PCovC classifier contains `{name}`, " + "which is not supported. The MultiOutputClassifier " + "must contain an instance of `" + f"{'`, `'.join(c.__name__ for c in compatible_clfs[:-1])}" + "`, or `precomputed`." + ) multioutput = self.n_outputs_ != 1 precomputed = self.classifier == "precomputed" diff --git a/tests/test_kernel_pcovc.py b/tests/test_kernel_pcovc.py index 725e25deb..9f29e4c60 100644 --- a/tests/test_kernel_pcovc.py +++ b/tests/test_kernel_pcovc.py @@ -260,9 +260,9 @@ def test_incompatible_classifier(self): str(cm.exception), "Classifier must be an instance of " "`LogisticRegression`, `LogisticRegressionCV`, `LinearSVC`, " - "`LinearDiscriminantAnalysis`, `MultiOutputClassifier`, `RidgeClassifier`, " - "`RidgeClassifierCV`, `SGDClassifier`, `Perceptron`, " - "or `precomputed`", + "`LinearDiscriminantAnalysis`, `RidgeClassifier`, `RidgeClassifierCV`, " + "`SGDClassifier`, `Perceptron`, `MultiOutputClassifier`, " + "or `precomputed`.", ) def test_none_classifier(self): @@ -590,7 +590,33 @@ def test_decision_function_multioutput(self): T = kpcovc.transform(self.X) _ = kpcovc.decision_function(T=T) - # TODO: Add tests for addition of score function to pcovc.py + def test_score(self): + """Check that KernelPCovC's score behaves properly with multiple labels.""" + kpcovc_multi = self.model( + classifier=MultiOutputClassifier(estimator=LogisticRegression()) + ) + kpcovc_multi.fit(self.X, np.column_stack((self.Y, self.Y))) + score_multi = kpcovc_multi.score(self.X, np.column_stack((self.Y, self.Y))) + + kpcovc_single = self.model().fit(self.X, self.Y) + score_single = kpcovc_single.score(self.X, self.Y) + self.assertEqual(score_single, score_multi) + + def test_bad_multioutput_estimator(self): + """Check that KernelPCovC returns an error when a MultiOutputClassifier + is improperly constructed. + """ + with self.assertRaises(ValueError) as cm: + pcovc = self.model(classifier=MultiOutputClassifier(estimator=GaussianNB())) + pcovc.fit(self.X, np.column_stack((self.Y, self.Y))) + self.assertEqual( + str(cm.exception), + "The instance of MultiOutputClassifier passed as the KernelPCovC classifier" + " contains `GaussianNB`, which is not supported. The MultiOutputClassifier " + "must contain an instance of `LogisticRegression`, `LogisticRegressionCV`, " + "`LinearSVC`, `LinearDiscriminantAnalysis`, `RidgeClassifier`, " + "`RidgeClassifierCV`, `SGDClassifier`, `Perceptron`, or `precomputed`.", + ) if __name__ == "__main__": diff --git a/tests/test_pcovc.py b/tests/test_pcovc.py index f5e0ba829..b8ae3365c 100644 --- a/tests/test_pcovc.py +++ b/tests/test_pcovc.py @@ -536,9 +536,9 @@ def test_incompatible_classifier(self): str(cm.exception), "Classifier must be an instance of " "`LogisticRegression`, `LogisticRegressionCV`, `LinearSVC`, " - "`LinearDiscriminantAnalysis`, `MultiOutputClassifier`, `RidgeClassifier`, " - "`RidgeClassifierCV`, `SGDClassifier`, `Perceptron`, " - "or `precomputed`", + "`LinearDiscriminantAnalysis`, `RidgeClassifier`, `RidgeClassifierCV`, " + "`SGDClassifier`, `Perceptron`, `MultiOutputClassifier`, " + "or `precomputed`.", ) def test_none_classifier(self): @@ -648,7 +648,9 @@ def test_decision_function_multioutput(self): """Check that PCovC's decision_function works in edge cases when `n_outputs_ > 1`. """ - pcovc = self.model(classifier=MultiOutputClassifier(estimator=LinearSVC())) + pcovc = self.model( + classifier=MultiOutputClassifier(estimator=LogisticRegression()) + ) pcovc.fit(self.X, np.column_stack((self.Y, self.Y))) with self.assertRaises(ValueError) as cm: _ = pcovc.decision_function() @@ -660,7 +662,33 @@ def test_decision_function_multioutput(self): T = pcovc.transform(self.X) _ = pcovc.decision_function(T=T) - # TODO: Add tests for addition of score function to pcovc.py + def test_score(self): + """Check that PCovC's score behaves properly with multiple labels.""" + pcovc_multi = self.model( + classifier=MultiOutputClassifier(estimator=LogisticRegression()) + ) + pcovc_multi.fit(self.X, np.column_stack((self.Y, self.Y))) + score_multi = pcovc_multi.score(self.X, np.column_stack((self.Y, self.Y))) + + pcovc_single = self.model().fit(self.X, self.Y) + score_single = pcovc_single.score(self.X, self.Y) + self.assertEqual(score_single, score_multi) + + def test_bad_multioutput_estimator(self): + """Check that PCovC returns an error when a MultiOutputClassifier + is improperly constructed. + """ + with self.assertRaises(ValueError) as cm: + pcovc = self.model(classifier=MultiOutputClassifier(estimator=GaussianNB())) + pcovc.fit(self.X, np.column_stack((self.Y, self.Y))) + self.assertEqual( + str(cm.exception), + "The instance of MultiOutputClassifier passed as the PCovC classifier " + "contains `GaussianNB`, which is not supported. The MultiOutputClassifier " + "must contain an instance of `LogisticRegression`, `LogisticRegressionCV`, " + "`LinearSVC`, `LinearDiscriminantAnalysis`, `RidgeClassifier`, " + "`RidgeClassifierCV`, `SGDClassifier`, `Perceptron`, or `precomputed`.", + ) if __name__ == "__main__": From 1ff4fedfe72f1d553a88fd495d49908f27c85612 Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Mon, 15 Sep 2025 15:56:18 -0500 Subject: [PATCH 18/27] Fixing ptz and pxz for multioutput --- examples/pcovc/PCovC_multioutput.py | 6 ++- src/skmatter/decomposition/_kernel_pcovc.py | 41 +++++++++++---------- src/skmatter/decomposition/_pcovc.py | 35 ++++++++---------- tests/test_pcovc.py | 1 + 4 files changed, 43 insertions(+), 40 deletions(-) diff --git a/examples/pcovc/PCovC_multioutput.py b/examples/pcovc/PCovC_multioutput.py index ddf08a4b0..b6cd00cb8 100644 --- a/examples/pcovc/PCovC_multioutput.py +++ b/examples/pcovc/PCovC_multioutput.py @@ -22,8 +22,8 @@ plt.rcParams["image.cmap"] = "tab10" plt.rcParams["scatter.edgecolors"] = "k" # %% -# -# +# For this, we will use the `sklearn.datasets.load_digits` dataset. +# This dataset contains 8x8 images of handwritten digits (0-9). X, y = load_digits(return_X_y=True) x_scaler = StandardScaler() X_scaled = StandardScaler().fit_transform(X) @@ -127,3 +127,5 @@ axs[1, 0].set_ylabel("PCovC") fig.colorbar(scat_pca, ax=axs, orientation="horizontal") fig.suptitle("Multiclass-Multilabel PCovC") + +# %% diff --git a/src/skmatter/decomposition/_kernel_pcovc.py b/src/skmatter/decomposition/_kernel_pcovc.py index 913ea9132..481593c73 100644 --- a/src/skmatter/decomposition/_kernel_pcovc.py +++ b/src/skmatter/decomposition/_kernel_pcovc.py @@ -39,8 +39,8 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): where :math:`\alpha` is a mixing parameter, :math:`\mathbf{K}` is the input kernel of shape :math:`(n_{samples}, n_{samples})` - and :math:`\mathbf{Z}` is a matrix of class confidence scores of shape - :math:`(n_{samples}, n_{classes})` + and :math:`\mathbf{Z}` is a tensor of class confidence scores of shape + :math:`(n_{samples}, n_{classes}, n_{labels})` Parameters ---------- @@ -82,10 +82,10 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): - ``sklearn.linear_model.LogisticRegressionCV()`` - ``sklearn.svm.LinearSVC()`` - ``sklearn.discriminant_analysis.LinearDiscriminantAnalysis()`` - - ``sklearn.multioutput.MultiOutputClassifier()`` + - ``sklearn.linear_model.Perceptron()`` - ``sklearn.linear_model.RidgeClassifier()`` - ``sklearn.linear_model.RidgeClassifierCV()`` - - ``sklearn.linear_model.Perceptron()`` + - ``sklearn.multioutput.MultiOutputClassifier()`` If a pre-fitted classifier is provided, it is used to compute :math:`{\mathbf{Z}}`. @@ -167,13 +167,15 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): the projector, or weights, from the input kernel :math:`\mathbf{K}` to the latent-space projection :math:`\mathbf{T}` - pkz_: numpy.ndarray of size :math:`({n_{samples}, })` or :math:`({n_{samples}, n_{classes}})` - the projector, or weights, from the input kernel :math:`\mathbf{K}` - to the class confidence scores :math:`\mathbf{Z}` + pkz_ : ndarray of size :math:`({n_{features}, {n_{classes}}})`, or list of + ndarrays of size :math:`({n_{features}, {n_{classes_i}}})` for a dataset + with :math: `i` labels. + the projector, or weights, from the input space :math:`\mathbf{X}` + to the class confidence scores :math:`\mathbf{Z}`. - ptz_: numpy.ndarray of size :math:`({n_{components}, })` or :math:`({n_{components}, n_{classes}})` - the projector, or weights, from the latent-space projection - :math:`\mathbf{T}` to the class confidence scores :math:`\mathbf{Z}` + ptz_ : ndarray of size :math:`({n_{components}, {n_{classes}}})`, or list of + ndarrays of size :math:`({n_{components}, {n_{classes_i}}})` for a dataset + with :math: `i` labels. ptx_: numpy.ndarray of size :math:`({n_{components}, n_{features}})` the projector, or weights, from the latent-space projection @@ -271,13 +273,16 @@ def fit(self, X, Y, W=None): scaled to have unit variance, otherwise :math:`\mathbf{X}` should be scaled so that each feature has a variance of 1 / n_features. - Y : numpy.ndarray, shape (n_samples,) - Training data, where n_samples is the number of samples. + Y : numpy.ndarray, shape (n_samples,) or (n_samples, n_outputs) + Training data, where n_samples is the number of samples and + n_outputs is the number of outputs. - W : numpy.ndarray, shape (n_features, n_classes) + W : numpy.ndarray, shape (n_features, n_classes) or (n_features, ) Classification weights, optional when classifier = `precomputed`. If not passed, it is assumed that the weights will be taken from a - linear classifier fit between K and Y. + linear classifier fit between :math:`\mathbf{X}` and :math:`\mathbf{Y}`. + In the multioutput case, use + `` W = np.hstack([est_.coef_.T for est_ in classifier.estimators_])``. Returns ------- @@ -355,7 +360,7 @@ def fit(self, X, Y, W=None): else: W = _.coef_.T - else: + elif W is None: self.z_classifier_ = check_cl_fit(classifier, K, Y) if multioutput: W = np.hstack([est_.coef_.T for est_ in self.z_classifier_.estimators_]) @@ -374,10 +379,8 @@ def fit(self, X, Y, W=None): self.classifier_ = clone(classifier).fit(K @ self.pkt_, Y) if multioutput: - self.ptz_ = np.hstack( - [est_.coef_.T for est_ in self.classifier_.estimators_] - ) - self.pkz_ = self.pkt_ @ self.ptz_ + self.ptz_ = [est_.coef_.T for est_ in self.classifier_.estimators_] + self.pkz_ = [self.pkt_ @ ptz for ptz in self.ptz_] else: self.ptz_ = self.classifier_.coef_.T self.pkz_ = self.pkt_ @ self.ptz_ diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index 8e618e55e..cfc221d9a 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -11,7 +11,6 @@ ) from sklearn.linear_model._base import LinearClassifierMixin -from sklearn.base import MultiOutputMixin from sklearn.multioutput import MultiOutputClassifier from sklearn.svm import LinearSVC from sklearn.utils import check_array @@ -36,8 +35,8 @@ class PCovC(LinearClassifierMixin, _BasePCov): (1 - \alpha) \mathbf{Z}\mathbf{Z}^T where :math:`\alpha` is a mixing parameter, :math:`\mathbf{X}` is an input matrix of shape - :math:`(n_{samples}, n_{features})`, and :math:`\mathbf{Z}` is a matrix of class confidence scores - of shape :math:`(n_{samples}, n_{classes})`. For :math:`(n_{samples} < n_{features})`, + :math:`(n_{samples}, n_{features})`, and :math:`\mathbf{Z}` is a tensor of class confidence scores + of shape :math:`(n_{samples}, n_{classes}, n_{labels})`. For :math:`(n_{samples} < n_{features})`, this can be more efficiently computed using the eigendecomposition of a modified covariance matrix :math:`\mathbf{\tilde{C}}` @@ -112,10 +111,10 @@ class PCovC(LinearClassifierMixin, _BasePCov): - ``sklearn.linear_model.LogisticRegressionCV()`` - ``sklearn.svm.LinearSVC()`` - ``sklearn.discriminant_analysis.LinearDiscriminantAnalysis()`` - - ``sklearn.multioutput.MultiOutputClassifier()`` + - ``sklearn.linear_model.Perceptron()`` - ``sklearn.linear_model.RidgeClassifier()`` - ``sklearn.linear_model.RidgeClassifierCV()`` - - ``sklearn.linear_model.Perceptron()`` + - ``sklearn.multioutput.MultiOutputClassifier()`` If a pre-fitted classifier is provided, it is used to compute :math:`{\mathbf{Z}}`. @@ -175,11 +174,15 @@ class PCovC(LinearClassifierMixin, _BasePCov): the projector, or weights, from the input space :math:`\mathbf{X}` to the latent-space projection :math:`\mathbf{T}` - pxz_ : ndarray of size :math:`({n_{features}, })`, :math:`({n_{features}, n_{classes}})` + pxz_ : ndarray of size :math:`({n_{features}, {n_{classes}}})`, or list of + ndarrays of size :math:`({n_{features}, {n_{classes_i}}})` for a dataset + with :math: `i` labels. the projector, or weights, from the input space :math:`\mathbf{X}` to the class confidence scores :math:`\mathbf{Z}`. - ptz_ : ndarray of size :math:`({n_{components}, })`, :math:`({n_{components}, n_{classes}})` + ptz_ : ndarray of size :math:`({n_{components}, {n_{classes}}})`, or list of + ndarrays of size :math:`({n_{components}, {n_{classes_i}}})` for a dataset + with :math: `i` labels. the projector, or weights, from from the latent-space projection :math:`\mathbf{T}` to the class confidence scores :math:`\mathbf{Z}`. @@ -267,7 +270,7 @@ def fit(self, X, Y, W=None): Classification weights, optional when classifier is ``precomputed``. If not passed, it is assumed that the weights will be taken from a linear classifier fit between :math:`\mathbf{X}` and :math:`\mathbf{Y}`. - In the multioutput case, + In the multioutput case, use `` W = np.hstack([est_.coef_.T for est_ in classifier.estimators_])``. """ X, Y = validate_data(self, X, Y, multi_output=True, y_numeric=False) @@ -329,7 +332,7 @@ def fit(self, X, Y, W=None): W = np.hstack([_.coef_.T for _ in _.estimators_]) else: W = _.coef_.T - else: + elif W is None: self.z_classifier_ = check_cl_fit(classifier, X, Y) if multioutput: W = np.hstack([est_.coef_.T for est_ in self.z_classifier_.estimators_]) @@ -337,7 +340,7 @@ def fit(self, X, Y, W=None): W = self.z_classifier_.coef_.T Z = X @ W - + if self.space_ == "feature": self._fit_feature_space(X, Y, Z) else: @@ -348,19 +351,12 @@ def fit(self, X, Y, W=None): self.classifier_ = clone(classifier).fit(X @ self.pxt_, Y) if multioutput: - self.ptz_ = np.hstack( - [est_.coef_.T for est_ in self.classifier_.estimators_] - ) - # print(f"pxt {self.pxt_.shape}") - # print(f"ptz {self.ptz_.shape}") - self.pxz_ = self.pxt_ @ self.ptz_ - # print(f"pxz {self.pxz_.shape}") + self.ptz_ = [est_.coef_.T for est_ in self.classifier_.estimators_] + self.pxz_ = [self.pxt_ @ ptz for ptz in self.ptz_] else: self.ptz_ = self.classifier_.coef_.T - # print(self.ptz_.shape) self.pxz_ = self.pxt_ @ self.ptz_ - # print(self.ptz_.shape) if not multioutput and type_of_target(Y) == "binary": self.pxz_ = self.pxz_.reshape( X.shape[1], @@ -531,3 +527,4 @@ def score(self, X, y, sample_weight=None): # Inherit the docstring from scikit-learn score.__doc__ = LinearClassifierMixin.score.__doc__ + \ No newline at end of file diff --git a/tests/test_pcovc.py b/tests/test_pcovc.py index b8ae3365c..166c5f1f4 100644 --- a/tests/test_pcovc.py +++ b/tests/test_pcovc.py @@ -606,6 +606,7 @@ def test_precomputed_multioutput(self): classifier.fit(self.X, Y_double) W = np.hstack([est_.coef_.T for est_ in classifier.estimators_]) + print(W.shape) pcovc1 = self.model(mixing=0.5, classifier="precomputed", n_components=1) pcovc1.fit(self.X, Y_double, W) t1 = pcovc1.transform(self.X) From 7b345f38e9ec110ba4e05cb292938260dae0d638 Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Mon, 15 Sep 2025 16:35:10 -0500 Subject: [PATCH 19/27] fix linting --- src/skmatter/decomposition/_pcovc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index cfc221d9a..ebe46cdbe 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -340,7 +340,7 @@ def fit(self, X, Y, W=None): W = self.z_classifier_.coef_.T Z = X @ W - + if self.space_ == "feature": self._fit_feature_space(X, Y, Z) else: @@ -527,4 +527,3 @@ def score(self, X, y, sample_weight=None): # Inherit the docstring from scikit-learn score.__doc__ = LinearClassifierMixin.score.__doc__ - \ No newline at end of file From c00b77a83e205e71f2a7cd86b0cc11dcb3fef833 Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Fri, 3 Oct 2025 11:48:58 -0500 Subject: [PATCH 20/27] Merge remote-tracking branch 'origin/main' into multioutput-pcovc --- .github/workflows/build.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/tests-dev.yml | 6 ++- .github/workflows/tests.yml | 11 +++-- examples/pcovc/KPCovC_Comparison.py | 6 ++- examples/pcovc/KPCovC_Hyperparameters.py | 6 ++- examples/pcovc/PCovC_Hyperparameters.py | 2 + src/skmatter/decomposition/_kernel_pcovc.py | 44 ++++++++++++++++- src/skmatter/decomposition/_pcovc.py | 54 ++++++++++++++++++++- tests/test_kernel_pcovc.py | 41 ++++++++++++++++ tests/test_pcovc.py | 50 ++++++++++++++++++- tests/test_sample_simple_cur.py | 2 +- 14 files changed, 212 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 61e753ea9..a170b1859 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1fa0d4420..d2468624c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -11,7 +11,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: setup Python uses: actions/setup-python@v5 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5ae3af316..c1fb9ef03 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 85f4c0eb5..e1aa66377 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: setup Python diff --git a/.github/workflows/tests-dev.yml b/.github/workflows/tests-dev.yml index be5f5f62f..7534227d5 100644 --- a/.github/workflows/tests-dev.yml +++ b/.github/workflows/tests-dev.yml @@ -10,10 +10,12 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.13"] + python-version: + - "3.10" + - "3.13" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index da175cea6..2518fea1d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,11 +11,16 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.10", "3.13"] + os: + - ubuntu-latest + - macos-latest + - windows-latest + python-version: + - "3.10" + - "3.13" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/examples/pcovc/KPCovC_Comparison.py b/examples/pcovc/KPCovC_Comparison.py index 5028a9b5c..f47cf9b4e 100644 --- a/examples/pcovc/KPCovC_Comparison.py +++ b/examples/pcovc/KPCovC_Comparison.py @@ -33,6 +33,7 @@ random_state = 0 n_components = 2 +scale_z = True # %% # @@ -85,7 +86,7 @@ # Both PCA and PCovC fail to produce linearly separable latent space # maps. We will need a kernel method to effectively separate the moon classes. -mixing = 0.10 +mixing = 0.5 alpha_d = 0.5 alpha_p = 0.4 @@ -95,6 +96,7 @@ n_components=n_components, random_state=random_state, mixing=mixing, + scale_z=scale_z, classifier=LinearSVC(), ): "PCovC", } @@ -138,6 +140,7 @@ random_state=random_state, mixing=mixing, center=center, + scale_z=scale_z, **kernel_params, ): {"title": "Kernel PCovC", "eps": 2}, } @@ -220,6 +223,7 @@ mixing=mixing, classifier=model, center=center, + scale_z=scale_z, **models[model]["kernel_params"], ) t_kpcovc_train = kpcovc.fit_transform(X_train_scaled, y_train) diff --git a/examples/pcovc/KPCovC_Hyperparameters.py b/examples/pcovc/KPCovC_Hyperparameters.py index ce3948e25..d848d70bb 100644 --- a/examples/pcovc/KPCovC_Hyperparameters.py +++ b/examples/pcovc/KPCovC_Hyperparameters.py @@ -65,7 +65,8 @@ fig, axs = plt.subplots(2, len(kernels), figsize=(len(kernels) * 4, 8)) center = True -mixing = 0.10 +mixing = 0.5 +scale_z = True for i, kernel in enumerate(kernels): kpca = KernelPCA( @@ -83,6 +84,7 @@ random_state=random_state, **kernel_params.get(kernel, {}), center=center, + scale_z=scale_z, ) t_kpcovc = kpcovc.fit_transform(X_scaled, y) @@ -118,7 +120,7 @@ kpcovc = KernelPCovC( n_components=n_components, random_state=random_state, - mixing=mixing, + mixing=0.1, center=center, kernel="rbf", gamma=gamma, diff --git a/examples/pcovc/PCovC_Hyperparameters.py b/examples/pcovc/PCovC_Hyperparameters.py index 22989a95d..427ba6dab 100644 --- a/examples/pcovc/PCovC_Hyperparameters.py +++ b/examples/pcovc/PCovC_Hyperparameters.py @@ -77,6 +77,7 @@ n_components=n_components, random_state=random_state, classifier=LogisticRegressionCV(), + scale_z=True, ) pcovc.fit(X_scaled, y) @@ -120,6 +121,7 @@ n_components=n_components, random_state=random_state, classifier=model, + scale_z=True, ) pcovc.fit(X_scaled, y) diff --git a/src/skmatter/decomposition/_kernel_pcovc.py b/src/skmatter/decomposition/_kernel_pcovc.py index 481593c73..1e0a5bab0 100644 --- a/src/skmatter/decomposition/_kernel_pcovc.py +++ b/src/skmatter/decomposition/_kernel_pcovc.py @@ -1,3 +1,4 @@ +import warnings import numpy as np from sklearn import clone @@ -17,7 +18,7 @@ from sklearn.linear_model._base import LinearClassifierMixin from sklearn.utils.multiclass import check_classification_targets, type_of_target -from skmatter.preprocessing import KernelNormalizer +from skmatter.preprocessing import KernelNormalizer, StandardFlexibleScaler from skmatter.utils import check_cl_fit from skmatter.decomposition import _BaseKPCov @@ -99,6 +100,9 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): constructed, with ``sklearn.linear_model.LogisticRegression()`` models used for each label. + scale_z: bool, default=False + Whether to scale Z prior to eigendecomposition. + kernel : {"linear", "poly", "rbf", "sigmoid", "precomputed"} or callable, default="linear" Kernel. @@ -129,6 +133,14 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): and for matrix inversions. Must be of range [0.0, infinity). + z_mean_tol: float, default=1e-12 + Tolerance for the column means of Z. + Must be of range [0.0, infinity). + + z_var_tol: float, default=1.5 + Tolerance for the column variances of Z. + Must be of range [0.0, infinity). + n_jobs : int, default=None The number of parallel jobs to run. :obj:`None` means 1 unless in a :obj:`joblib.parallel_backend` context. @@ -185,6 +197,9 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): The data used to fit the model. This attribute is used to build kernels from new data. + scale_z: bool + Whether Z is being scaled prior to eigendecomposition. + Examples -------- >>> import numpy as np @@ -192,7 +207,7 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): >>> from sklearn.preprocessing import StandardScaler >>> X = np.array([[-2, 3, -1, 0], [2, 0, -3, 1], [3, 0, -1, 3], [2, -2, 1, 0]]) >>> X = StandardScaler().fit_transform(X) - >>> Y = np.array([[2], [0], [1], [2]]) + >>> Y = np.array([2, 0, 1, 2]) >>> kpcovc = KernelPCovC( ... mixing=0.1, ... n_components=2, @@ -218,6 +233,7 @@ def __init__( n_components=None, svd_solver="auto", classifier=None, + scale_z=False, kernel="linear", gamma=None, degree=3, @@ -226,6 +242,8 @@ def __init__( center=False, fit_inverse_transform=False, tol=1e-12, + z_mean_tol=1e-12, + z_var_tol=1.5, n_jobs=None, iterated_power="auto", random_state=None, @@ -247,6 +265,9 @@ def __init__( fit_inverse_transform=fit_inverse_transform, ) self.classifier = classifier + self.scale_z = scale_z + self.z_mean_tol = z_mean_tol + self.z_var_tol = z_var_tol def fit(self, X, Y, W=None): r"""Fit the model with X and Y. @@ -368,6 +389,25 @@ def fit(self, X, Y, W=None): W = self.z_classifier_.coef_.T Z = K @ W + if self.scale_z: + Z = StandardFlexibleScaler().fit_transform(Z) + + z_means_ = np.mean(Z, axis=0) + z_vars_ = np.var(Z, axis=0) + + if np.max(np.abs(z_means_)) > self.z_mean_tol: + warnings.warn( + "This class does not automatically center Z, and the column means " + "of Z are greater than the supplied tolerance. We recommend scaling " + "Z (and the weights) by setting `scale_z=True`." + ) + + if np.max(z_vars_) > self.z_var_tol: + warnings.warn( + "This class does not automatically scale Z, and the column variances " + "of Z are greater than the supplied tolerance. We recommend scaling " + "Z (and the weights) by setting `scale_z=True`." + ) self._fit(K, Z, W) diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index ebe46cdbe..00915bdf1 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -18,6 +18,8 @@ from sklearn.utils.validation import check_is_fitted, validate_data from skmatter.decomposition import _BasePCov from skmatter.utils import check_cl_fit +from skmatter.preprocessing import StandardFlexibleScaler +import warnings class PCovC(LinearClassifierMixin, _BasePCov): @@ -98,6 +100,14 @@ class PCovC(LinearClassifierMixin, _BasePCov): Tolerance for singular values computed by svd_solver == 'arpack'. Must be of range [0.0, infinity). + z_mean_tol: float, default=1e-12 + Tolerance for the column means of Z. + Must be of range [0.0, infinity). + + z_var_tol: float, default=1.5 + Tolerance for the column variances of Z. + Must be of range [0.0, infinity). + space: {'feature', 'sample', 'auto'}, default='auto' whether to compute the PCovC in ``sample`` or ``feature`` space. The default is equal to ``sample`` when :math:`{n_{samples} < n_{features}}` @@ -128,6 +138,9 @@ class PCovC(LinearClassifierMixin, _BasePCov): constructed, with ``sklearn.linear_model.LogisticRegression()`` models used for each label. + scale_z: bool, default=False + Whether to scale Z prior to eigendecomposition. + iterated_power : int or 'auto', default='auto' Number of iterations for the power method computed by svd_solver == 'randomized'. @@ -148,6 +161,14 @@ class PCovC(LinearClassifierMixin, _BasePCov): Tolerance for singular values computed by svd_solver == 'arpack'. Must be of range [0.0, infinity). + z_mean_tol: float + Tolerance for the column means of Z. + Must be of range [0.0, infinity). + + z_var_tol: float + Tolerance for the column variances of Z. + Must be of range [0.0, infinity). + space: {'feature', 'sample', 'auto'}, default='auto' whether to compute the PCovC in ``sample`` or ``feature`` space. The default is equal to ``sample`` when :math:`{n_{samples} < n_{features}}` @@ -186,6 +207,9 @@ class PCovC(LinearClassifierMixin, _BasePCov): the projector, or weights, from from the latent-space projection :math:`\mathbf{T}` to the class confidence scores :math:`\mathbf{Z}`. + scale_z: bool + Whether Z is being scaled prior to eigendecomposition + explained_variance_ : numpy.ndarray of shape (n_components,) The amount of variance explained by each of the selected components. Equal to n_components largest eigenvalues @@ -220,8 +244,11 @@ def __init__( n_components=None, svd_solver="auto", tol=1e-12, + z_mean_tol=1e-12, + z_var_tol=1.5, space="auto", classifier=None, + scale_z=False, iterated_power="auto", random_state=None, whiten=False, @@ -237,6 +264,9 @@ def __init__( whiten=whiten, ) self.classifier = classifier + self.scale_z = scale_z + self.z_mean_tol = z_mean_tol + self.z_var_tol = z_var_tol def fit(self, X, Y, W=None): r"""Fit the model with X and Y. @@ -337,10 +367,32 @@ def fit(self, X, Y, W=None): if multioutput: W = np.hstack([est_.coef_.T for est_ in self.z_classifier_.estimators_]) else: - W = self.z_classifier_.coef_.T + W = self.z_classifier_.coef_.T.copy() Z = X @ W + if self.scale_z: + z_scaler = StandardFlexibleScaler().fit(Z) + Z = z_scaler.transform(Z) + W /= z_scaler.scale_.reshape(1, -1) + + z_means_ = np.mean(Z, axis=0) + z_vars_ = np.var(Z, axis=0) + + if np.max(np.abs(z_means_)) > self.z_mean_tol: + warnings.warn( + "This class does not automatically center Z, and the column means " + "of Z are greater than the supplied tolerance. We recommend scaling " + "Z (and the weights) by setting `scale_z=True`." + ) + + if np.max(z_vars_) > self.z_var_tol: + warnings.warn( + "This class does not automatically scale Z, and the column variances " + "of Z are greater than the supplied tolerance. We recommend scaling " + "Z (and the weights) by setting `scale_z=True`." + ) + if self.space_ == "feature": self._fit_feature_space(X, Y, Z) else: diff --git a/tests/test_kernel_pcovc.py b/tests/test_kernel_pcovc.py index 9f29e4c60..3a28bfa76 100644 --- a/tests/test_kernel_pcovc.py +++ b/tests/test_kernel_pcovc.py @@ -1,4 +1,5 @@ import unittest +import warnings import numpy as np from sklearn import exceptions @@ -35,10 +36,12 @@ def __init__(self, *args, **kwargs): lambda mixing=0.5, classifier=LogisticRegression(), n_components=4, + scale_z=True, **kwargs: KernelPCovC( mixing=mixing, classifier=classifier, n_components=n_components, + scale_z=scale_z, svd_solver=kwargs.pop("svd_solver", "full"), **kwargs, ) @@ -331,6 +334,44 @@ def test_precomputed_classification(self): self.assertTrue(np.linalg.norm(t3 - t2) < self.error_tol) self.assertTrue(np.linalg.norm(t3 - t1) < self.error_tol) + def test_scale_z_parameter(self): + """Check that changing scale_z changes the eigendecomposition.""" + kpcovc_scaled = self.model(scale_z=True) + kpcovc_scaled.fit(self.X, self.Y) + + kpcovc_unscaled = self.model(scale_z=False) + kpcovc_unscaled.fit(self.X, self.Y) + assert not np.allclose(kpcovc_scaled.pkt_, kpcovc_unscaled.pkt_) + + def test_z_scaling(self): + """ + Check that KPCovC raises a warning if Z is not of scale, and does not + if it is. + """ + kpcovc = self.model(n_components=2, scale_z=True) + + with warnings.catch_warnings(): + kpcovc.fit(self.X, self.Y) + warnings.simplefilter("error") + self.assertEqual(1 + 1, 2) + + kpcovc = self.model(n_components=2, scale_z=False, z_mean_tol=0, z_var_tol=0) + + with warnings.catch_warnings(record=True) as w: + kpcovc.fit(self.X, self.Y) + self.assertEqual( + str(w[0].message), + "This class does not automatically center Z, and the column means " + "of Z are greater than the supplied tolerance. We recommend scaling " + "Z (and the weights) by setting `scale_z=True`.", + ) + self.assertEqual( + str(w[1].message), + "This class does not automatically scale Z, and the column variances " + "of Z are greater than the supplied tolerance. We recommend scaling " + "Z (and the weights) by setting `scale_z=True`.", + ) + class KernelTests(KernelPCovCBaseTest): def test_kernel_types(self): diff --git a/tests/test_pcovc.py b/tests/test_pcovc.py index 166c5f1f4..baf49d44f 100644 --- a/tests/test_pcovc.py +++ b/tests/test_pcovc.py @@ -20,8 +20,11 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.model = ( - lambda mixing=0.5, classifier=LogisticRegression(), **kwargs: PCovC( - mixing=mixing, classifier=classifier, **kwargs + lambda mixing=0.5, + classifier=LogisticRegression(), + scale_z=True, + **kwargs: PCovC( + mixing=mixing, classifier=classifier, scale_z=scale_z, **kwargs ) ) @@ -404,6 +407,35 @@ def test_centering(self): "mean is greater than the supplied tolerance.", ) + def test_z_scaling(self): + """ + Check that PCovC raises a warning if Z is not of scale, and does not + if it is. + """ + pcovc = self.model(n_components=2, scale_z=True) + + with warnings.catch_warnings(): + pcovc.fit(self.X, self.Y) + warnings.simplefilter("error") + self.assertEqual(1 + 1, 2) + + pcovc = self.model(n_components=2, scale_z=False, z_mean_tol=0, z_var_tol=0) + + with warnings.catch_warnings(record=True) as w: + pcovc.fit(self.X, self.Y) + self.assertEqual( + str(w[0].message), + "This class does not automatically center Z, and the column means " + "of Z are greater than the supplied tolerance. We recommend scaling " + "Z (and the weights) by setting `scale_z=True`.", + ) + self.assertEqual( + str(w[1].message), + "This class does not automatically scale Z, and the column variances " + "of Z are greater than the supplied tolerance. We recommend scaling " + "Z (and the weights) by setting `scale_z=True`.", + ) + def test_T_shape(self): """Check that PCovC returns a latent space projection consistent with the shape of the input matrix. @@ -466,6 +498,9 @@ def test_default_ncomponents(self): self.assertEqual(pcovc.n_components_, min(self.X.shape)) def test_prefit_classifier(self): + """Check that a passed prefit classifier is not modified in + PCovC's `fit` call. + """ classifier = LinearSVC() classifier.fit(self.X, self.Y) pcovc = self.model(mixing=0.5, classifier=classifier) @@ -577,6 +612,17 @@ def test_incompatible_coef_shape(self): % (len(pcovc_multi.classes_), self.X.shape[1], cl_binary.coef_.shape), ) + def test_scale_z_parameter(self): + """Check that changing scale_z changes the eigendecomposition.""" + pcovc_scaled = self.model(scale_z=True) + pcovc_scaled.fit(self.X, self.Y) + + pcovc_unscaled = self.model(scale_z=False) + pcovc_unscaled.fit(self.X, self.Y) + assert not np.allclose( + pcovc_scaled.singular_values_, pcovc_unscaled.singular_values_ + ) + class PCovCMultiOutputTest(PCovCBaseTest): def test_prefit_multioutput(self): diff --git a/tests/test_sample_simple_cur.py b/tests/test_sample_simple_cur.py index 1aa9e5857..50885aedd 100644 --- a/tests/test_sample_simple_cur.py +++ b/tests/test_sample_simple_cur.py @@ -1,7 +1,7 @@ import unittest import numpy as np -from sklearn.datasets import fetch_california_housing as load +from sklearn.datasets import load_diabetes as load from skmatter.sample_selection import CUR, FPS From 35c1b0fefbb79a38cb3374a07877934787d6b15f Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Mon, 6 Oct 2025 10:47:38 -0500 Subject: [PATCH 21/27] fixing docs --- src/skmatter/decomposition/_kernel_pcovc.py | 8 ++++---- src/skmatter/decomposition/_pcovc.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/skmatter/decomposition/_kernel_pcovc.py b/src/skmatter/decomposition/_kernel_pcovc.py index 1e0a5bab0..185be8b94 100644 --- a/src/skmatter/decomposition/_kernel_pcovc.py +++ b/src/skmatter/decomposition/_kernel_pcovc.py @@ -303,7 +303,7 @@ def fit(self, X, Y, W=None): not passed, it is assumed that the weights will be taken from a linear classifier fit between :math:`\mathbf{X}` and :math:`\mathbf{Y}`. In the multioutput case, use - `` W = np.hstack([est_.coef_.T for est_ in classifier.estimators_])``. + ``W = np.hstack([est_.coef_.T for est_ in classifier.estimators_])``. Returns ------- @@ -514,11 +514,11 @@ def decision_function(self, X=None, T=None): Returns ------- - Z : numpy.ndarray, shape (n_samples,) or (n_samples, n_classes), or a list of \ - n_outputs_ such arrays if n_outputs_ > 1 + Z : numpy.ndarray, shape (n_samples,) or (n_samples, n_classes), or + a list of n_outputs such arrays if n_outputs > 1. Confidence scores. For binary classification, has shape `(n_samples,)`, for multiclass classification, has shape `(n_samples, n_classes)`. - If n_outputs_ > 1, the list can contain arrays with differing shapes + If n_outputs > 1, the list can contain arrays with differing shapes depending on the number of classes in each output of Y. """ check_is_fitted(self, attributes=["pkz_", "ptz_"]) diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index 00915bdf1..9ad62d58a 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -133,8 +133,8 @@ class PCovC(LinearClassifierMixin, _BasePCov): `sklearn.pipeline.Pipeline` with model caching. In such cases, the classifier will be re-fitted on the same training data as the composite estimator. - If None and ``n_outputs < 2``, ``sklearn.linear_model.LogisticRegression()`` is used. - If None and ``n_outputs >= 2``, a ``sklearn.multioutput.MultiOutputClassifier()`` is + If None and ``n_outputs_ < 2``, ``sklearn.linear_model.LogisticRegression()`` is used. + If None and ``n_outputs_ >= 2``, a ``sklearn.multioutput.MultiOutputClassifier()`` is constructed, with ``sklearn.linear_model.LogisticRegression()`` models used for each label. @@ -301,7 +301,7 @@ def fit(self, X, Y, W=None): not passed, it is assumed that the weights will be taken from a linear classifier fit between :math:`\mathbf{X}` and :math:`\mathbf{Y}`. In the multioutput case, use - `` W = np.hstack([est_.coef_.T for est_ in classifier.estimators_])``. + ``W = np.hstack([est_.coef_.T for est_ in classifier.estimators_])``. """ X, Y = validate_data(self, X, Y, multi_output=True, y_numeric=False) From ff58c8d5f337333c605597d2c0d3da6ac8c6ce3e Mon Sep 17 00:00:00 2001 From: Christian Jorgensen <114787994+cajchristian@users.noreply.github.com> Date: Mon, 6 Oct 2025 11:06:41 -0500 Subject: [PATCH 22/27] Deleting duplicate scale_z --- src/skmatter/decomposition/_pcovc.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index 0964776e0..47fb57a41 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -141,9 +141,6 @@ class PCovC(LinearClassifierMixin, _BasePCov): scale_z: bool, default=False Whether to scale Z prior to eigendecomposition. - scale_z: bool, default=False - Whether to scale Z prior to eigendecomposition. - iterated_power : int or 'auto', default='auto' Number of iterations for the power method computed by svd_solver == 'randomized'. From 5fdba2a93b609ec8dd74daec1108395e04382a7a Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Mon, 6 Oct 2025 11:11:05 -0500 Subject: [PATCH 23/27] Deleting duplicate again --- src/skmatter/decomposition/_kernel_pcovc.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/skmatter/decomposition/_kernel_pcovc.py b/src/skmatter/decomposition/_kernel_pcovc.py index 800b8237d..185be8b94 100644 --- a/src/skmatter/decomposition/_kernel_pcovc.py +++ b/src/skmatter/decomposition/_kernel_pcovc.py @@ -103,9 +103,6 @@ class KernelPCovC(LinearClassifierMixin, _BaseKPCov): scale_z: bool, default=False Whether to scale Z prior to eigendecomposition. - scale_z: bool, default=False - Whether to scale Z prior to eigendecomposition. - kernel : {"linear", "poly", "rbf", "sigmoid", "precomputed"} or callable, default="linear" Kernel. From 314e6d3aa37627f22e686a3b2a14d973aecdb077 Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Mon, 6 Oct 2025 11:18:57 -0500 Subject: [PATCH 24/27] Deleting another duplicate --- src/skmatter/decomposition/_pcovc.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/skmatter/decomposition/_pcovc.py b/src/skmatter/decomposition/_pcovc.py index 47fb57a41..9ad62d58a 100644 --- a/src/skmatter/decomposition/_pcovc.py +++ b/src/skmatter/decomposition/_pcovc.py @@ -210,9 +210,6 @@ class PCovC(LinearClassifierMixin, _BasePCov): scale_z: bool Whether Z is being scaled prior to eigendecomposition - scale_z: bool - Whether Z is being scaled prior to eigendecomposition - explained_variance_ : numpy.ndarray of shape (n_components,) The amount of variance explained by each of the selected components. Equal to n_components largest eigenvalues From cdb33b5f37e48750548084f7dcaf989fe7de0bef Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Mon, 6 Oct 2025 11:33:46 -0500 Subject: [PATCH 25/27] Fixing a test with kpcovc --- tests/test_kernel_pcovc.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_kernel_pcovc.py b/tests/test_kernel_pcovc.py index 3a28bfa76..74dc1a2b0 100644 --- a/tests/test_kernel_pcovc.py +++ b/tests/test_kernel_pcovc.py @@ -5,6 +5,7 @@ from sklearn import exceptions from sklearn.svm import LinearSVC from sklearn.datasets import load_breast_cancer as get_dataset +from sklearn.datasets import load_iris as get_multiclass_dataset from sklearn.multioutput import MultiOutputClassifier from sklearn.naive_bayes import GaussianNB from sklearn.utils.validation import check_X_y @@ -633,14 +634,16 @@ def test_decision_function_multioutput(self): def test_score(self): """Check that KernelPCovC's score behaves properly with multiple labels.""" + X, y = get_multiclass_dataset(return_X_y=True) + X = StandardScaler().fit_transform(X) kpcovc_multi = self.model( classifier=MultiOutputClassifier(estimator=LogisticRegression()) ) - kpcovc_multi.fit(self.X, np.column_stack((self.Y, self.Y))) - score_multi = kpcovc_multi.score(self.X, np.column_stack((self.Y, self.Y))) + kpcovc_multi.fit(X, np.column_stack((y, y))) + score_multi = kpcovc_multi.score(X, np.column_stack((y, y))) - kpcovc_single = self.model().fit(self.X, self.Y) - score_single = kpcovc_single.score(self.X, self.Y) + kpcovc_single = self.model().fit(X, y) + score_single = kpcovc_single.score(X, y) self.assertEqual(score_single, score_multi) def test_bad_multioutput_estimator(self): From 701b6caec7bc34ab65c1d4a20dae696b0d1ff37e Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Mon, 6 Oct 2025 11:56:49 -0500 Subject: [PATCH 26/27] Fixing opacity in example --- examples/pcovc/KPCovC_Comparison.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/examples/pcovc/KPCovC_Comparison.py b/examples/pcovc/KPCovC_Comparison.py index f47cf9b4e..c055b0ead 100644 --- a/examples/pcovc/KPCovC_Comparison.py +++ b/examples/pcovc/KPCovC_Comparison.py @@ -88,7 +88,8 @@ mixing = 0.5 alpha_d = 0.5 -alpha_p = 0.4 +alpha_train = 0.2 +alpha_test = 0.8 models = { PCA(n_components=n_components): "PCA", @@ -106,9 +107,10 @@ for ax, model in zip(axs, models): t_train = model.fit_transform(X_train_scaled, y_train) t_test = model.transform(X_test_scaled) - - ax.scatter(t_test[:, 0], t_test[:, 1], alpha=alpha_p, cmap=cm_bright, c=y_test) - ax.scatter(t_train[:, 0], t_train[:, 1], cmap=cm_bright, c=y_train) + + ax.scatter(t_train[:, 0], t_train[:, 1], alpha=alpha_train, cmap=cm_bright, c=y_train) + ax.scatter(t_test[:, 0], t_test[:, 1], alpha=alpha_test, cmap=cm_bright, c=y_test) + ax.set_title(models[model]) plt.tight_layout() @@ -166,8 +168,8 @@ eps=models[model]["eps"], grid_resolution=resolution, ) - ax.scatter(t_test[:, 0], t_test[:, 1], alpha=alpha_p, cmap=cm_bright, c=y_test) - ax.scatter(t_train[:, 0], t_train[:, 1], cmap=cm_bright, c=y_train) + ax.scatter(t_train[:, 0], t_train[:, 1], alpha=alpha_train, cmap=cm_bright, c=y_train) + ax.scatter(t_test[:, 0], t_test[:, 1], alpha=alpha_test, cmap=cm_bright, c=y_test) ax.set_title(models[model]["title"]) ax.text( @@ -240,15 +242,17 @@ eps=models[model].get("eps", 1), grid_resolution=resolution, ) - + + ax.scatter(t_kpcovc_train[:, 0], t_kpcovc_train[:, 1], alpha=alpha_train, cmap=cm_bright, c=y_train) + ax.scatter( t_kpcovc_test[:, 0], t_kpcovc_test[:, 1], cmap=cm_bright, - alpha=alpha_p, + alpha=alpha_test, c=y_test, ) - ax.scatter(t_kpcovc_train[:, 0], t_kpcovc_train[:, 1], cmap=cm_bright, c=y_train) + ax.text( 0.70, 0.03, From 156fa22feabd875075b59e4ee173d683060a6a0c Mon Sep 17 00:00:00 2001 From: Christian Jorgensen Date: Mon, 6 Oct 2025 11:57:09 -0500 Subject: [PATCH 27/27] Fix linting --- examples/pcovc/KPCovC_Comparison.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/examples/pcovc/KPCovC_Comparison.py b/examples/pcovc/KPCovC_Comparison.py index c055b0ead..6f5d6aa26 100644 --- a/examples/pcovc/KPCovC_Comparison.py +++ b/examples/pcovc/KPCovC_Comparison.py @@ -107,10 +107,11 @@ for ax, model in zip(axs, models): t_train = model.fit_transform(X_train_scaled, y_train) t_test = model.transform(X_test_scaled) - - ax.scatter(t_train[:, 0], t_train[:, 1], alpha=alpha_train, cmap=cm_bright, c=y_train) + + ax.scatter( + t_train[:, 0], t_train[:, 1], alpha=alpha_train, cmap=cm_bright, c=y_train + ) ax.scatter(t_test[:, 0], t_test[:, 1], alpha=alpha_test, cmap=cm_bright, c=y_test) - ax.set_title(models[model]) plt.tight_layout() @@ -168,7 +169,9 @@ eps=models[model]["eps"], grid_resolution=resolution, ) - ax.scatter(t_train[:, 0], t_train[:, 1], alpha=alpha_train, cmap=cm_bright, c=y_train) + ax.scatter( + t_train[:, 0], t_train[:, 1], alpha=alpha_train, cmap=cm_bright, c=y_train + ) ax.scatter(t_test[:, 0], t_test[:, 1], alpha=alpha_test, cmap=cm_bright, c=y_test) ax.set_title(models[model]["title"]) @@ -242,9 +245,15 @@ eps=models[model].get("eps", 1), grid_resolution=resolution, ) - - ax.scatter(t_kpcovc_train[:, 0], t_kpcovc_train[:, 1], alpha=alpha_train, cmap=cm_bright, c=y_train) - + + ax.scatter( + t_kpcovc_train[:, 0], + t_kpcovc_train[:, 1], + alpha=alpha_train, + cmap=cm_bright, + c=y_train, + ) + ax.scatter( t_kpcovc_test[:, 0], t_kpcovc_test[:, 1], @@ -252,7 +261,7 @@ alpha=alpha_test, c=y_test, ) - + ax.text( 0.70, 0.03,