From acb50041d52cec6fbd9db87b7a6964c8e52c1565 Mon Sep 17 00:00:00 2001 From: Nwanna-Joseph <98966754+Nwanna-Joseph@users.noreply.github.com> Date: Tue, 31 May 2022 16:39:32 +0000 Subject: [PATCH 01/35] Simplify comparison --- sklearn/decomposition/_pca.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/decomposition/_pca.py b/sklearn/decomposition/_pca.py index 635e119ae445d..d2ef71ee7caa5 100644 --- a/sklearn/decomposition/_pca.py +++ b/sklearn/decomposition/_pca.py @@ -472,7 +472,7 @@ def _fit(self, X): # Small problem or n_components == 'mle', just call full PCA if max(X.shape) <= 500 or n_components == "mle": self._fit_svd_solver = "full" - elif n_components >= 1 and n_components < 0.8 * min(X.shape): + elif 1 <= n_components < 0.8 * min(X.shape): self._fit_svd_solver = "randomized" # This is also the case of n_components in (0,1) else: From 863fcafbdce245accd16d1af2b13f61aafde1b62 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Thu, 2 Jun 2022 12:09:26 +0000 Subject: [PATCH 02/35] impl --- sklearn/linear_model/_stochastic_gradient.py | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index a4c129d101ef1..d8168d7a19445 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -22,6 +22,9 @@ from ..utils.extmath import safe_sparse_dot from ..utils.multiclass import _check_partial_fit_first_call from ..utils.validation import check_is_fitted, _check_sample_weight +from ..utils._param_validation import Interval +from ..utils._param_validation import StrOptions +from ..utils._param_validation import validate_params from ..utils.fixes import delayed from ..exceptions import ConvergenceWarning from ..model_selection import StratifiedShuffleSplit, ShuffleSplit @@ -76,6 +79,28 @@ def __call__(self, coef, intercept): class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): """Base class for SGD classification and regression.""" + _parameter_constraints = { + "penalty": [StrOptions({"l2", "l1", "elasticnet","none"}), None], #done + "alpha": [ Interval(Integral, 0, None, closed="left")], #done + "C": [Interval(Integral, 1, None, closed="left")], + "l1_ratio": [Interval(Real, 0.0, 1.0, closed="both")], + "fit_intercept": [bool], + "max_iter": [Interval(Real,0.0,1.0,closed="neither"), None], + "tol": ["random_state"], + "shuffle": [bool], #done + "verbose": ["random_state"], + "epsilon": ["random_state"], + "random_state": ["random_state"], + "learning_rate": [StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}, internal={"optimal"})], + "eta0": ["random_state"], + "power_t": ["random_state"], + "early_stopping": [bool], #done + "validation_fraction": [Interval(Real, 0.0, 1.0, closed="neither")], + "n_iter_no_change": [Interval(Integer, 1, None, closed="left")], + "warm_start": [bool], + "average": [bool], + } + def __init__( self, loss, From c10120a12bc3821b557b1e360bf54f043bafd651 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Thu, 2 Jun 2022 13:06:06 +0000 Subject: [PATCH 03/35] setup _parameter_constraints --- sklearn/linear_model/_stochastic_gradient.py | 454 +++++++++---------- 1 file changed, 226 insertions(+), 228 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index d8168d7a19445..26ed77b34d252 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -80,50 +80,50 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): """Base class for SGD classification and regression.""" _parameter_constraints = { - "penalty": [StrOptions({"l2", "l1", "elasticnet","none"}), None], #done - "alpha": [ Interval(Integral, 0, None, closed="left")], #done - "C": [Interval(Integral, 1, None, closed="left")], - "l1_ratio": [Interval(Real, 0.0, 1.0, closed="both")], - "fit_intercept": [bool], - "max_iter": [Interval(Real,0.0,1.0,closed="neither"), None], - "tol": ["random_state"], - "shuffle": [bool], #done - "verbose": ["random_state"], - "epsilon": ["random_state"], + "penalty": [StrOptions({"l2", "l1", "elasticnet"})], # done + "alpha": [Interval(Real, 0, None, closed="left")], # done + "C": [Interval(Integral, 1, None, closed="left")], # redundant? + "l1_ratio": [Interval(Real, 0, 1, closed="both")], # done + "fit_intercept": [bool], # done + "max_iter": [Interval(Integer, 1, None, closed="left")], # done + "tol": [Interval(Real,0,None,closed="left")], # done + "shuffle": [bool], # done + "verbose": [Interval(Integer,0,None,closed="left")], + "epsilon": [Interval(Real,0,None,closed="left")], "random_state": ["random_state"], - "learning_rate": [StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}, internal={"optimal"})], - "eta0": ["random_state"], - "power_t": ["random_state"], - "early_stopping": [bool], #done - "validation_fraction": [Interval(Real, 0.0, 1.0, closed="neither")], + "learning_rate": [StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"})], + "eta0": [Interval(Real,0,None,closed="neither")], + "power_t": [Interval(Real, None,None,closed="neither")], + "early_stopping": [bool], # done + "validation_fraction": [Interval(Real, 0, 1, closed="neither")], "n_iter_no_change": [Interval(Integer, 1, None, closed="left")], "warm_start": [bool], - "average": [bool], + "average": [Interval(Integer, 1, None, closed="left"),bool], } def __init__( - self, - loss, - *, - penalty="l2", - alpha=0.0001, - C=1.0, - l1_ratio=0.15, - fit_intercept=True, - max_iter=1000, - tol=1e-3, - shuffle=True, - verbose=0, - epsilon=0.1, - random_state=None, - learning_rate="optimal", - eta0=0.0, - power_t=0.5, - early_stopping=False, - validation_fraction=0.1, - n_iter_no_change=5, - warm_start=False, - average=False, + self, + loss, + *, + penalty="l2", + alpha=0.0001, + C=1.0, + l1_ratio=0.15, + fit_intercept=True, + max_iter=1000, + tol=1e-3, + shuffle=True, + verbose=0, + epsilon=0.1, + random_state=None, + learning_rate="optimal", + eta0=0.0, + power_t=0.5, + early_stopping=False, + validation_fraction=0.1, + n_iter_no_change=5, + warm_start=False, + average=False, ): self.loss = loss self.penalty = penalty @@ -228,7 +228,7 @@ def _get_penalty_type(self, penalty): raise ValueError("Penalty %s is not supported. " % penalty) from e def _allocate_parameter_mem( - self, n_classes, n_features, coef_init=None, intercept_init=None, one_class=0 + self, n_classes, n_features, coef_init=None, intercept_init=None, one_class=0 ): """Allocate mem for parameters; initialize if provided.""" if n_classes > 2: @@ -339,7 +339,7 @@ def _make_validation_split(self, y): return validation_mask def _make_validation_score_cb( - self, validation_mask, X, y, sample_weight, classes=None + self, validation_mask, X, y, sample_weight, classes=None ): if not self.early_stopping: return None @@ -386,19 +386,19 @@ def _prepare_fit_binary(est, y, i): def fit_binary( - est, - i, - X, - y, - alpha, - C, - learning_rate, - max_iter, - pos_weight, - neg_weight, - sample_weight, - validation_mask=None, - random_state=None, + est, + i, + X, + y, + alpha, + C, + learning_rate, + max_iter, + pos_weight, + neg_weight, + sample_weight, + validation_mask=None, + random_state=None, ): """Fit a single binary classifier. @@ -520,7 +520,6 @@ def fit_binary( class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta): - # TODO(1.2): Remove "squared_loss" # TODO(1.3): Remove "log"" loss_functions = { @@ -539,29 +538,29 @@ class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta): @abstractmethod def __init__( - self, - loss="hinge", - *, - penalty="l2", - alpha=0.0001, - l1_ratio=0.15, - fit_intercept=True, - max_iter=1000, - tol=1e-3, - shuffle=True, - verbose=0, - epsilon=DEFAULT_EPSILON, - n_jobs=None, - random_state=None, - learning_rate="optimal", - eta0=0.0, - power_t=0.5, - early_stopping=False, - validation_fraction=0.1, - n_iter_no_change=5, - class_weight=None, - warm_start=False, - average=False, + self, + loss="hinge", + *, + penalty="l2", + alpha=0.0001, + l1_ratio=0.15, + fit_intercept=True, + max_iter=1000, + tol=1e-3, + shuffle=True, + verbose=0, + epsilon=DEFAULT_EPSILON, + n_jobs=None, + random_state=None, + learning_rate="optimal", + eta0=0.0, + power_t=0.5, + early_stopping=False, + validation_fraction=0.1, + n_iter_no_change=5, + class_weight=None, + warm_start=False, + average=False, ): super().__init__( @@ -589,18 +588,18 @@ def __init__( self.n_jobs = n_jobs def _partial_fit( - self, - X, - y, - alpha, - C, - loss, - learning_rate, - max_iter, - classes, - sample_weight, - coef_init, - intercept_init, + self, + X, + y, + alpha, + C, + loss, + learning_rate, + max_iter, + classes, + sample_weight, + coef_init, + intercept_init, ): first_call = not hasattr(self, "classes_") X, y = self._validate_data( @@ -669,16 +668,16 @@ def _partial_fit( return self def _fit( - self, - X, - y, - alpha, - C, - loss, - learning_rate, - coef_init=None, - intercept_init=None, - sample_weight=None, + self, + X, + y, + alpha, + C, + loss, + learning_rate, + coef_init=None, + intercept_init=None, + sample_weight=None, ): self._validate_params() if hasattr(self, "classes_"): @@ -723,9 +722,9 @@ def _fit( ) if ( - self.tol is not None - and self.tol > -np.inf - and self.n_iter_ == self.max_iter + self.tol is not None + and self.tol > -np.inf + and self.n_iter_ == self.max_iter ): warnings.warn( "Maximum number of iteration reached before " @@ -1193,29 +1192,29 @@ class SGDClassifier(BaseSGDClassifier): """ def __init__( - self, - loss="hinge", - *, - penalty="l2", - alpha=0.0001, - l1_ratio=0.15, - fit_intercept=True, - max_iter=1000, - tol=1e-3, - shuffle=True, - verbose=0, - epsilon=DEFAULT_EPSILON, - n_jobs=None, - random_state=None, - learning_rate="optimal", - eta0=0.0, - power_t=0.5, - early_stopping=False, - validation_fraction=0.1, - n_iter_no_change=5, - class_weight=None, - warm_start=False, - average=False, + self, + loss="hinge", + *, + penalty="l2", + alpha=0.0001, + l1_ratio=0.15, + fit_intercept=True, + max_iter=1000, + tol=1e-3, + shuffle=True, + verbose=0, + epsilon=DEFAULT_EPSILON, + n_jobs=None, + random_state=None, + learning_rate="optimal", + eta0=0.0, + power_t=0.5, + early_stopping=False, + validation_fraction=0.1, + n_iter_no_change=5, + class_weight=None, + warm_start=False, + average=False, ): super().__init__( loss=loss, @@ -1368,7 +1367,6 @@ def _more_tags(self): class BaseSGDRegressor(RegressorMixin, BaseSGD): - # TODO: Remove squared_loss in v1.2 loss_functions = { "squared_error": (SquaredLoss,), @@ -1380,27 +1378,27 @@ class BaseSGDRegressor(RegressorMixin, BaseSGD): @abstractmethod def __init__( - self, - loss="squared_error", - *, - penalty="l2", - alpha=0.0001, - l1_ratio=0.15, - fit_intercept=True, - max_iter=1000, - tol=1e-3, - shuffle=True, - verbose=0, - epsilon=DEFAULT_EPSILON, - random_state=None, - learning_rate="invscaling", - eta0=0.01, - power_t=0.25, - early_stopping=False, - validation_fraction=0.1, - n_iter_no_change=5, - warm_start=False, - average=False, + self, + loss="squared_error", + *, + penalty="l2", + alpha=0.0001, + l1_ratio=0.15, + fit_intercept=True, + max_iter=1000, + tol=1e-3, + shuffle=True, + verbose=0, + epsilon=DEFAULT_EPSILON, + random_state=None, + learning_rate="invscaling", + eta0=0.01, + power_t=0.25, + early_stopping=False, + validation_fraction=0.1, + n_iter_no_change=5, + warm_start=False, + average=False, ): super().__init__( loss=loss, @@ -1425,17 +1423,17 @@ def __init__( ) def _partial_fit( - self, - X, - y, - alpha, - C, - loss, - learning_rate, - max_iter, - sample_weight, - coef_init, - intercept_init, + self, + X, + y, + alpha, + C, + loss, + learning_rate, + max_iter, + sample_weight, + coef_init, + intercept_init, ): first_call = getattr(self, "coef_", None) is None X, y = self._validate_data( @@ -1507,16 +1505,16 @@ def partial_fit(self, X, y, sample_weight=None): ) def _fit( - self, - X, - y, - alpha, - C, - loss, - learning_rate, - coef_init=None, - intercept_init=None, - sample_weight=None, + self, + X, + y, + alpha, + C, + loss, + learning_rate, + coef_init=None, + intercept_init=None, + sample_weight=None, ): self._validate_params() if self.warm_start and getattr(self, "coef_", None) is not None: @@ -1545,9 +1543,9 @@ def _fit( ) if ( - self.tol is not None - and self.tol > -np.inf - and self.n_iter_ == self.max_iter + self.tol is not None + and self.tol > -np.inf + and self.n_iter_ == self.max_iter ): warnings.warn( "Maximum number of iteration reached before " @@ -1630,7 +1628,7 @@ def predict(self, X): return self._decision_function(X) def _fit_regressor( - self, X, y, alpha, C, loss, learning_rate, sample_weight, max_iter + self, X, y, alpha, C, loss, learning_rate, sample_weight, max_iter ): loss_function = self._get_loss_function(loss) penalty_type = self._get_penalty_type(self.penalty) @@ -1937,27 +1935,27 @@ class SGDRegressor(BaseSGDRegressor): """ def __init__( - self, - loss="squared_error", - *, - penalty="l2", - alpha=0.0001, - l1_ratio=0.15, - fit_intercept=True, - max_iter=1000, - tol=1e-3, - shuffle=True, - verbose=0, - epsilon=DEFAULT_EPSILON, - random_state=None, - learning_rate="invscaling", - eta0=0.01, - power_t=0.25, - early_stopping=False, - validation_fraction=0.1, - n_iter_no_change=5, - warm_start=False, - average=False, + self, + loss="squared_error", + *, + penalty="l2", + alpha=0.0001, + l1_ratio=0.15, + fit_intercept=True, + max_iter=1000, + tol=1e-3, + shuffle=True, + verbose=0, + epsilon=DEFAULT_EPSILON, + random_state=None, + learning_rate="invscaling", + eta0=0.01, + power_t=0.25, + early_stopping=False, + validation_fraction=0.1, + n_iter_no_change=5, + warm_start=False, + average=False, ): super().__init__( loss=loss, @@ -2134,19 +2132,19 @@ class SGDOneClassSVM(BaseSGD, OutlierMixin): loss_functions = {"hinge": (Hinge, 1.0)} def __init__( - self, - nu=0.5, - fit_intercept=True, - max_iter=1000, - tol=1e-3, - shuffle=True, - verbose=0, - random_state=None, - learning_rate="optimal", - eta0=0.0, - power_t=0.5, - warm_start=False, - average=False, + self, + nu=0.5, + fit_intercept=True, + max_iter=1000, + tol=1e-3, + shuffle=True, + verbose=0, + random_state=None, + learning_rate="optimal", + eta0=0.0, + power_t=0.5, + warm_start=False, + average=False, ): alpha = nu / 2 @@ -2278,16 +2276,16 @@ def _fit_one_class(self, X, alpha, C, sample_weight, learning_rate, max_iter): self.offset_ = 1 - np.atleast_1d(intercept) def _partial_fit( - self, - X, - alpha, - C, - loss, - learning_rate, - max_iter, - sample_weight, - coef_init, - offset_init, + self, + X, + alpha, + C, + loss, + learning_rate, + max_iter, + sample_weight, + coef_init, + offset_init, ): first_call = getattr(self, "coef_", None) is None X = self._validate_data( @@ -2372,15 +2370,15 @@ def partial_fit(self, X, y=None, sample_weight=None): ) def _fit( - self, - X, - alpha, - C, - loss, - learning_rate, - coef_init=None, - offset_init=None, - sample_weight=None, + self, + X, + alpha, + C, + loss, + learning_rate, + coef_init=None, + offset_init=None, + sample_weight=None, ): self._validate_params() @@ -2409,9 +2407,9 @@ def _fit( ) if ( - self.tol is not None - and self.tol > -np.inf - and self.n_iter_ == self.max_iter + self.tol is not None + and self.tol > -np.inf + and self.n_iter_ == self.max_iter ): warnings.warn( "Maximum number of iteration reached before " From f22ab2abbf64f4005f2651625cf2fe43f683cfe3 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Thu, 2 Jun 2022 13:07:00 +0000 Subject: [PATCH 04/35] bug fixes --- sklearn/linear_model/_stochastic_gradient.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 26ed77b34d252..83529917a97f5 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -7,6 +7,7 @@ """ import numpy as np +from numbers import Integral, Real import warnings from abc import ABCMeta, abstractmethod @@ -85,10 +86,10 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "C": [Interval(Integral, 1, None, closed="left")], # redundant? "l1_ratio": [Interval(Real, 0, 1, closed="both")], # done "fit_intercept": [bool], # done - "max_iter": [Interval(Integer, 1, None, closed="left")], # done + "max_iter": [Interval(Integral, 1, None, closed="left")], # done "tol": [Interval(Real,0,None,closed="left")], # done "shuffle": [bool], # done - "verbose": [Interval(Integer,0,None,closed="left")], + "verbose": [Interval(Integral,0,None,closed="left")], "epsilon": [Interval(Real,0,None,closed="left")], "random_state": ["random_state"], "learning_rate": [StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"})], @@ -96,9 +97,9 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "power_t": [Interval(Real, None,None,closed="neither")], "early_stopping": [bool], # done "validation_fraction": [Interval(Real, 0, 1, closed="neither")], - "n_iter_no_change": [Interval(Integer, 1, None, closed="left")], + "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], "warm_start": [bool], - "average": [Interval(Integer, 1, None, closed="left"),bool], + "average": [Interval(Integral, 1, None, closed="left"),bool], } def __init__( From 7b7670b51d0cd5f720dc2b33dc0fcaf5a714d765 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Thu, 2 Jun 2022 21:45:44 +0000 Subject: [PATCH 05/35] _parameter_constraints for BaseSGD --- sklearn/linear_model/_stochastic_gradient.py | 36 +++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 83529917a97f5..140b58d78fa70 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -81,25 +81,27 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): """Base class for SGD classification and regression.""" _parameter_constraints = { - "penalty": [StrOptions({"l2", "l1", "elasticnet"})], # done - "alpha": [Interval(Real, 0, None, closed="left")], # done + "penalty": [StrOptions({"l2", "l1", "elasticnet"})], + "alpha": [Interval(Real, 0, None, closed="left")], "C": [Interval(Integral, 1, None, closed="left")], # redundant? - "l1_ratio": [Interval(Real, 0, 1, closed="both")], # done - "fit_intercept": [bool], # done - "max_iter": [Interval(Integral, 1, None, closed="left")], # done - "tol": [Interval(Real,0,None,closed="left")], # done - "shuffle": [bool], # done - "verbose": [Interval(Integral,0,None,closed="left")], - "epsilon": [Interval(Real,0,None,closed="left")], + "l1_ratio": [Interval(Real, 0, 1, closed="both")], + "fit_intercept": [bool], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "tol": [Interval(Real, 0, None, closed="left")], + "shuffle": [bool], + "verbose": [Interval(Integral, 0, None, closed="left")], + "epsilon": [Interval(Real, 0, None, closed="left")], "random_state": ["random_state"], - "learning_rate": [StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"})], - "eta0": [Interval(Real,0,None,closed="neither")], - "power_t": [Interval(Real, None,None,closed="neither")], + "learning_rate": [ + StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}) + ], + "eta0": [Interval(Real, 0, None, closed="neither")], + "power_t": [Interval(Real, None, None, closed="neither")], "early_stopping": [bool], # done "validation_fraction": [Interval(Real, 0, 1, closed="neither")], "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], "warm_start": [bool], - "average": [Interval(Integral, 1, None, closed="left"),bool], + "average": [Interval(Integral, 1, None, closed="left"), bool], } def __init__( @@ -537,6 +539,14 @@ class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta): "squared_epsilon_insensitive": (SquaredEpsilonInsensitive, DEFAULT_EPSILON), } + _parameter_constraints = { + "loss": [StrOptions({"hinge", "squared_hinge", "perceptron", "log_loss", "log", "modified_huber", + "squared_error", "squared_loss", "huber", "epsilon_insensitive", + "squared_epsilon_insensitive"})], + "n_jobs": [Interval(Integral,None,None,closed="neither"),None], + "class_weight": [StrOptions({"balanced"}), None] # a bit tricky? + } + @abstractmethod def __init__( self, From 8525f1294f720e1ff79b6c05edad17eaf8f760ab Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Sat, 4 Jun 2022 11:35:44 +0000 Subject: [PATCH 06/35] fixes --- sklearn/linear_model/_stochastic_gradient.py | 23 +++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 140b58d78fa70..ecdc0fd0b1519 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -81,7 +81,8 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): """Base class for SGD classification and regression.""" _parameter_constraints = { - "penalty": [StrOptions({"l2", "l1", "elasticnet"})], + "loss": [], + "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None], "alpha": [Interval(Real, 0, None, closed="left")], "C": [Interval(Integral, 1, None, closed="left")], # redundant? "l1_ratio": [Interval(Real, 0, 1, closed="both")], @@ -93,11 +94,11 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "epsilon": [Interval(Real, 0, None, closed="left")], "random_state": ["random_state"], "learning_rate": [ - StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}) + StrOptions({"constant", "optimal", "invscaling", "adaptive"}, internal={"pa1","pa2"}) ], "eta0": [Interval(Real, 0, None, closed="neither")], "power_t": [Interval(Real, None, None, closed="neither")], - "early_stopping": [bool], # done + "early_stopping": [bool], "validation_fraction": [Interval(Real, 0, 1, closed="neither")], "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], "warm_start": [bool], @@ -153,8 +154,9 @@ def __init__( def fit(self, X, y): """Fit model.""" - def _validate_params(self, for_partial_fit=False): + def _revalidate_params(self, for_partial_fit=False): """Validate input params.""" + if not isinstance(self.shuffle, bool): raise ValueError("shuffle must be either True or False") if not isinstance(self.early_stopping, bool): @@ -388,6 +390,7 @@ def _prepare_fit_binary(est, y, i): return y_i, coef, intercept, average_coef, average_intercept +@validate_params() def fit_binary( est, i, @@ -540,11 +543,12 @@ class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta): } _parameter_constraints = { + **BaseSGD._parameter_constraints, "loss": [StrOptions({"hinge", "squared_hinge", "perceptron", "log_loss", "log", "modified_huber", "squared_error", "squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive"})], - "n_jobs": [Interval(Integral,None,None,closed="neither"),None], - "class_weight": [StrOptions({"balanced"}), None] # a bit tricky? + "n_jobs": [None, Integral], + "class_weight": [StrOptions({"balanced"}), None] # a bit tricky? } @abstractmethod @@ -612,6 +616,7 @@ def _partial_fit( coef_init, intercept_init, ): + self._validate_params() first_call = not hasattr(self, "classes_") X, y = self._validate_data( X, @@ -690,7 +695,7 @@ def _fit( intercept_init=None, sample_weight=None, ): - self._validate_params() + self._revalidate_params() if hasattr(self, "classes_"): # delete the attribute otherwise _partial_fit thinks it's not the first call delattr(self, "classes_") @@ -867,7 +872,7 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): self : object Returns an instance of self. """ - self._validate_params(for_partial_fit=True) + self._revalidate_params(for_partial_fit=True) if self.class_weight in ["balanced"]: raise ValueError( "class_weight '{0}' is not supported for " @@ -1528,6 +1533,7 @@ def _fit( sample_weight=None, ): self._validate_params() + if self.warm_start and getattr(self, "coef_", None) is not None: if coef_init is None: coef_init = self.coef_ @@ -1592,6 +1598,7 @@ def fit(self, X, y, coef_init=None, intercept_init=None, sample_weight=None): self : object Fitted `SGDRegressor` estimator. """ + return self._fit( X, y, From 5744e4f228c0a9c600f890e3510a1f55f6f9b207 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Sat, 4 Jun 2022 13:00:51 +0000 Subject: [PATCH 07/35] fix validation --- sklearn/linear_model/_stochastic_gradient.py | 46 +++----------------- 1 file changed, 5 insertions(+), 41 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index ecdc0fd0b1519..0c9fd77202ca3 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -87,7 +87,7 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "C": [Interval(Integral, 1, None, closed="left")], # redundant? "l1_ratio": [Interval(Real, 0, 1, closed="both")], "fit_intercept": [bool], - "max_iter": [Interval(Integral, 1, None, closed="left")], + "max_iter": [Interval(Integral, 1, None, closed="left")], # None "tol": [Interval(Real, 0, None, closed="left")], "shuffle": [bool], "verbose": [Interval(Integral, 0, None, closed="left")], @@ -156,26 +156,10 @@ def fit(self, X, y): def _revalidate_params(self, for_partial_fit=False): """Validate input params.""" + self._validate_params() - if not isinstance(self.shuffle, bool): - raise ValueError("shuffle must be either True or False") - if not isinstance(self.early_stopping, bool): - raise ValueError("early_stopping must be either True or False") if self.early_stopping and for_partial_fit: raise ValueError("early_stopping should be False with partial_fit") - if self.max_iter is not None and self.max_iter <= 0: - raise ValueError("max_iter must be > zero. Got %f" % self.max_iter) - if not (0.0 <= self.l1_ratio <= 1.0): - raise ValueError("l1_ratio must be in [0, 1]") - if not isinstance(self, SGDOneClassSVM) and self.alpha < 0.0: - raise ValueError("alpha must be >= 0") - if self.n_iter_no_change < 1: - raise ValueError("n_iter_no_change must be >= 1") - if not (0.0 < self.validation_fraction < 1.0): - raise ValueError("validation_fraction must be in range (0, 1)") - if self.learning_rate in ("constant", "invscaling", "adaptive"): - if self.eta0 <= 0.0: - raise ValueError("eta0 must be > 0") if self.learning_rate == "optimal" and self.alpha == 0: raise ValueError( "alpha must be > 0 since " @@ -187,25 +171,6 @@ def _revalidate_params(self, for_partial_fit=False): self._get_penalty_type(self.penalty) self._get_learning_rate_type(self.learning_rate) - if self.loss not in self.loss_functions: - raise ValueError("The loss %s is not supported. " % self.loss) - - # TODO(1.2): remove "squared_loss" - if self.loss == "squared_loss": - warnings.warn( - "The loss 'squared_loss' was deprecated in v1.0 and will be " - "removed in version 1.2. Use `loss='squared_error'` which is " - "equivalent.", - FutureWarning, - ) - # TODO(1.3): remove "log" - if self.loss == "log": - warnings.warn( - "The loss 'log' was deprecated in v1.1 and will be removed in version " - "1.3. Use `loss='log_loss'` which is equivalent.", - FutureWarning, - ) - def _get_loss_function(self, loss): """Get concrete ``LossFunction`` object for str ``loss``.""" try: @@ -546,7 +511,7 @@ class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta): **BaseSGD._parameter_constraints, "loss": [StrOptions({"hinge", "squared_hinge", "perceptron", "log_loss", "log", "modified_huber", "squared_error", "squared_loss", "huber", "epsilon_insensitive", - "squared_epsilon_insensitive"})], + "squared_epsilon_insensitive"}, deprecated={"squared_loss","log"})], "n_jobs": [None, Integral], "class_weight": [StrOptions({"balanced"}), None] # a bit tricky? } @@ -616,7 +581,6 @@ def _partial_fit( coef_init, intercept_init, ): - self._validate_params() first_call = not hasattr(self, "classes_") X, y = self._validate_data( X, @@ -1506,7 +1470,7 @@ def partial_fit(self, X, y, sample_weight=None): self : object Returns an instance of self. """ - self._validate_params(for_partial_fit=True) + self._revalidate_params(for_partial_fit=True) return self._partial_fit( X, y, @@ -1532,7 +1496,7 @@ def _fit( intercept_init=None, sample_weight=None, ): - self._validate_params() + self._revalidate_params() if self.warm_start and getattr(self, "coef_", None) is not None: if coef_init is None: From 07722e5473c3ce54406ce1a8aaddb02bbd353cdb Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Sat, 4 Jun 2022 13:01:28 +0000 Subject: [PATCH 08/35] black linting --- sklearn/linear_model/_stochastic_gradient.py | 452 ++++++++++--------- 1 file changed, 235 insertions(+), 217 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 0c9fd77202ca3..da44ba2e66d79 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -84,17 +84,20 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "loss": [], "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None], "alpha": [Interval(Real, 0, None, closed="left")], - "C": [Interval(Integral, 1, None, closed="left")], # redundant? + "C": [Interval(Integral, 1, None, closed="left")], "l1_ratio": [Interval(Real, 0, 1, closed="both")], "fit_intercept": [bool], - "max_iter": [Interval(Integral, 1, None, closed="left")], # None + "max_iter": [Interval(Integral, 1, None, closed="left")], # None "tol": [Interval(Real, 0, None, closed="left")], "shuffle": [bool], "verbose": [Interval(Integral, 0, None, closed="left")], "epsilon": [Interval(Real, 0, None, closed="left")], "random_state": ["random_state"], "learning_rate": [ - StrOptions({"constant", "optimal", "invscaling", "adaptive"}, internal={"pa1","pa2"}) + StrOptions( + {"constant", "optimal", "invscaling", "adaptive"}, + internal={"pa1", "pa2"}, + ) ], "eta0": [Interval(Real, 0, None, closed="neither")], "power_t": [Interval(Real, None, None, closed="neither")], @@ -106,28 +109,28 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): } def __init__( - self, - loss, - *, - penalty="l2", - alpha=0.0001, - C=1.0, - l1_ratio=0.15, - fit_intercept=True, - max_iter=1000, - tol=1e-3, - shuffle=True, - verbose=0, - epsilon=0.1, - random_state=None, - learning_rate="optimal", - eta0=0.0, - power_t=0.5, - early_stopping=False, - validation_fraction=0.1, - n_iter_no_change=5, - warm_start=False, - average=False, + self, + loss, + *, + penalty="l2", + alpha=0.0001, + C=1.0, + l1_ratio=0.15, + fit_intercept=True, + max_iter=1000, + tol=1e-3, + shuffle=True, + verbose=0, + epsilon=0.1, + random_state=None, + learning_rate="optimal", + eta0=0.0, + power_t=0.5, + early_stopping=False, + validation_fraction=0.1, + n_iter_no_change=5, + warm_start=False, + average=False, ): self.loss = loss self.penalty = penalty @@ -198,7 +201,7 @@ def _get_penalty_type(self, penalty): raise ValueError("Penalty %s is not supported. " % penalty) from e def _allocate_parameter_mem( - self, n_classes, n_features, coef_init=None, intercept_init=None, one_class=0 + self, n_classes, n_features, coef_init=None, intercept_init=None, one_class=0 ): """Allocate mem for parameters; initialize if provided.""" if n_classes > 2: @@ -309,7 +312,7 @@ def _make_validation_split(self, y): return validation_mask def _make_validation_score_cb( - self, validation_mask, X, y, sample_weight, classes=None + self, validation_mask, X, y, sample_weight, classes=None ): if not self.early_stopping: return None @@ -357,19 +360,19 @@ def _prepare_fit_binary(est, y, i): @validate_params() def fit_binary( - est, - i, - X, - y, - alpha, - C, - learning_rate, - max_iter, - pos_weight, - neg_weight, - sample_weight, - validation_mask=None, - random_state=None, + est, + i, + X, + y, + alpha, + C, + learning_rate, + max_iter, + pos_weight, + neg_weight, + sample_weight, + validation_mask=None, + random_state=None, ): """Fit a single binary classifier. @@ -509,38 +512,53 @@ class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta): _parameter_constraints = { **BaseSGD._parameter_constraints, - "loss": [StrOptions({"hinge", "squared_hinge", "perceptron", "log_loss", "log", "modified_huber", - "squared_error", "squared_loss", "huber", "epsilon_insensitive", - "squared_epsilon_insensitive"}, deprecated={"squared_loss","log"})], + "loss": [ + StrOptions( + { + "hinge", + "squared_hinge", + "perceptron", + "log_loss", + "log", + "modified_huber", + "squared_error", + "squared_loss", + "huber", + "epsilon_insensitive", + "squared_epsilon_insensitive", + }, + deprecated={"squared_loss", "log"}, + ) + ], "n_jobs": [None, Integral], - "class_weight": [StrOptions({"balanced"}), None] # a bit tricky? + "class_weight": [StrOptions({"balanced"}), None], # a bit tricky? } @abstractmethod def __init__( - self, - loss="hinge", - *, - penalty="l2", - alpha=0.0001, - l1_ratio=0.15, - fit_intercept=True, - max_iter=1000, - tol=1e-3, - shuffle=True, - verbose=0, - epsilon=DEFAULT_EPSILON, - n_jobs=None, - random_state=None, - learning_rate="optimal", - eta0=0.0, - power_t=0.5, - early_stopping=False, - validation_fraction=0.1, - n_iter_no_change=5, - class_weight=None, - warm_start=False, - average=False, + self, + loss="hinge", + *, + penalty="l2", + alpha=0.0001, + l1_ratio=0.15, + fit_intercept=True, + max_iter=1000, + tol=1e-3, + shuffle=True, + verbose=0, + epsilon=DEFAULT_EPSILON, + n_jobs=None, + random_state=None, + learning_rate="optimal", + eta0=0.0, + power_t=0.5, + early_stopping=False, + validation_fraction=0.1, + n_iter_no_change=5, + class_weight=None, + warm_start=False, + average=False, ): super().__init__( @@ -568,18 +586,18 @@ def __init__( self.n_jobs = n_jobs def _partial_fit( - self, - X, - y, - alpha, - C, - loss, - learning_rate, - max_iter, - classes, - sample_weight, - coef_init, - intercept_init, + self, + X, + y, + alpha, + C, + loss, + learning_rate, + max_iter, + classes, + sample_weight, + coef_init, + intercept_init, ): first_call = not hasattr(self, "classes_") X, y = self._validate_data( @@ -648,16 +666,16 @@ def _partial_fit( return self def _fit( - self, - X, - y, - alpha, - C, - loss, - learning_rate, - coef_init=None, - intercept_init=None, - sample_weight=None, + self, + X, + y, + alpha, + C, + loss, + learning_rate, + coef_init=None, + intercept_init=None, + sample_weight=None, ): self._revalidate_params() if hasattr(self, "classes_"): @@ -702,9 +720,9 @@ def _fit( ) if ( - self.tol is not None - and self.tol > -np.inf - and self.n_iter_ == self.max_iter + self.tol is not None + and self.tol > -np.inf + and self.n_iter_ == self.max_iter ): warnings.warn( "Maximum number of iteration reached before " @@ -1172,29 +1190,29 @@ class SGDClassifier(BaseSGDClassifier): """ def __init__( - self, - loss="hinge", - *, - penalty="l2", - alpha=0.0001, - l1_ratio=0.15, - fit_intercept=True, - max_iter=1000, - tol=1e-3, - shuffle=True, - verbose=0, - epsilon=DEFAULT_EPSILON, - n_jobs=None, - random_state=None, - learning_rate="optimal", - eta0=0.0, - power_t=0.5, - early_stopping=False, - validation_fraction=0.1, - n_iter_no_change=5, - class_weight=None, - warm_start=False, - average=False, + self, + loss="hinge", + *, + penalty="l2", + alpha=0.0001, + l1_ratio=0.15, + fit_intercept=True, + max_iter=1000, + tol=1e-3, + shuffle=True, + verbose=0, + epsilon=DEFAULT_EPSILON, + n_jobs=None, + random_state=None, + learning_rate="optimal", + eta0=0.0, + power_t=0.5, + early_stopping=False, + validation_fraction=0.1, + n_iter_no_change=5, + class_weight=None, + warm_start=False, + average=False, ): super().__init__( loss=loss, @@ -1358,27 +1376,27 @@ class BaseSGDRegressor(RegressorMixin, BaseSGD): @abstractmethod def __init__( - self, - loss="squared_error", - *, - penalty="l2", - alpha=0.0001, - l1_ratio=0.15, - fit_intercept=True, - max_iter=1000, - tol=1e-3, - shuffle=True, - verbose=0, - epsilon=DEFAULT_EPSILON, - random_state=None, - learning_rate="invscaling", - eta0=0.01, - power_t=0.25, - early_stopping=False, - validation_fraction=0.1, - n_iter_no_change=5, - warm_start=False, - average=False, + self, + loss="squared_error", + *, + penalty="l2", + alpha=0.0001, + l1_ratio=0.15, + fit_intercept=True, + max_iter=1000, + tol=1e-3, + shuffle=True, + verbose=0, + epsilon=DEFAULT_EPSILON, + random_state=None, + learning_rate="invscaling", + eta0=0.01, + power_t=0.25, + early_stopping=False, + validation_fraction=0.1, + n_iter_no_change=5, + warm_start=False, + average=False, ): super().__init__( loss=loss, @@ -1403,17 +1421,17 @@ def __init__( ) def _partial_fit( - self, - X, - y, - alpha, - C, - loss, - learning_rate, - max_iter, - sample_weight, - coef_init, - intercept_init, + self, + X, + y, + alpha, + C, + loss, + learning_rate, + max_iter, + sample_weight, + coef_init, + intercept_init, ): first_call = getattr(self, "coef_", None) is None X, y = self._validate_data( @@ -1485,16 +1503,16 @@ def partial_fit(self, X, y, sample_weight=None): ) def _fit( - self, - X, - y, - alpha, - C, - loss, - learning_rate, - coef_init=None, - intercept_init=None, - sample_weight=None, + self, + X, + y, + alpha, + C, + loss, + learning_rate, + coef_init=None, + intercept_init=None, + sample_weight=None, ): self._revalidate_params() @@ -1524,9 +1542,9 @@ def _fit( ) if ( - self.tol is not None - and self.tol > -np.inf - and self.n_iter_ == self.max_iter + self.tol is not None + and self.tol > -np.inf + and self.n_iter_ == self.max_iter ): warnings.warn( "Maximum number of iteration reached before " @@ -1610,7 +1628,7 @@ def predict(self, X): return self._decision_function(X) def _fit_regressor( - self, X, y, alpha, C, loss, learning_rate, sample_weight, max_iter + self, X, y, alpha, C, loss, learning_rate, sample_weight, max_iter ): loss_function = self._get_loss_function(loss) penalty_type = self._get_penalty_type(self.penalty) @@ -1917,27 +1935,27 @@ class SGDRegressor(BaseSGDRegressor): """ def __init__( - self, - loss="squared_error", - *, - penalty="l2", - alpha=0.0001, - l1_ratio=0.15, - fit_intercept=True, - max_iter=1000, - tol=1e-3, - shuffle=True, - verbose=0, - epsilon=DEFAULT_EPSILON, - random_state=None, - learning_rate="invscaling", - eta0=0.01, - power_t=0.25, - early_stopping=False, - validation_fraction=0.1, - n_iter_no_change=5, - warm_start=False, - average=False, + self, + loss="squared_error", + *, + penalty="l2", + alpha=0.0001, + l1_ratio=0.15, + fit_intercept=True, + max_iter=1000, + tol=1e-3, + shuffle=True, + verbose=0, + epsilon=DEFAULT_EPSILON, + random_state=None, + learning_rate="invscaling", + eta0=0.01, + power_t=0.25, + early_stopping=False, + validation_fraction=0.1, + n_iter_no_change=5, + warm_start=False, + average=False, ): super().__init__( loss=loss, @@ -2114,19 +2132,19 @@ class SGDOneClassSVM(BaseSGD, OutlierMixin): loss_functions = {"hinge": (Hinge, 1.0)} def __init__( - self, - nu=0.5, - fit_intercept=True, - max_iter=1000, - tol=1e-3, - shuffle=True, - verbose=0, - random_state=None, - learning_rate="optimal", - eta0=0.0, - power_t=0.5, - warm_start=False, - average=False, + self, + nu=0.5, + fit_intercept=True, + max_iter=1000, + tol=1e-3, + shuffle=True, + verbose=0, + random_state=None, + learning_rate="optimal", + eta0=0.0, + power_t=0.5, + warm_start=False, + average=False, ): alpha = nu / 2 @@ -2258,16 +2276,16 @@ def _fit_one_class(self, X, alpha, C, sample_weight, learning_rate, max_iter): self.offset_ = 1 - np.atleast_1d(intercept) def _partial_fit( - self, - X, - alpha, - C, - loss, - learning_rate, - max_iter, - sample_weight, - coef_init, - offset_init, + self, + X, + alpha, + C, + loss, + learning_rate, + max_iter, + sample_weight, + coef_init, + offset_init, ): first_call = getattr(self, "coef_", None) is None X = self._validate_data( @@ -2352,15 +2370,15 @@ def partial_fit(self, X, y=None, sample_weight=None): ) def _fit( - self, - X, - alpha, - C, - loss, - learning_rate, - coef_init=None, - offset_init=None, - sample_weight=None, + self, + X, + alpha, + C, + loss, + learning_rate, + coef_init=None, + offset_init=None, + sample_weight=None, ): self._validate_params() @@ -2389,9 +2407,9 @@ def _fit( ) if ( - self.tol is not None - and self.tol > -np.inf - and self.n_iter_ == self.max_iter + self.tol is not None + and self.tol > -np.inf + and self.n_iter_ == self.max_iter ): warnings.warn( "Maximum number of iteration reached before " From 489f060b96fc4be600dfc69dd01693fcc29d7a65 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Sun, 5 Jun 2022 00:58:29 +0000 Subject: [PATCH 09/35] fixes --- sklearn/linear_model/_stochastic_gradient.py | 59 ++++++++++++-------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index da44ba2e66d79..c9c052b20547e 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -25,7 +25,6 @@ from ..utils.validation import check_is_fitted, _check_sample_weight from ..utils._param_validation import Interval from ..utils._param_validation import StrOptions -from ..utils._param_validation import validate_params from ..utils.fixes import delayed from ..exceptions import ConvergenceWarning from ..model_selection import StratifiedShuffleSplit, ShuffleSplit @@ -87,8 +86,8 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "C": [Interval(Integral, 1, None, closed="left")], "l1_ratio": [Interval(Real, 0, 1, closed="both")], "fit_intercept": [bool], - "max_iter": [Interval(Integral, 1, None, closed="left")], # None - "tol": [Interval(Real, 0, None, closed="left")], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "tol": [Interval(Real, 0, None, closed="left"), None], "shuffle": [bool], "verbose": [Interval(Integral, 0, None, closed="left")], "epsilon": [Interval(Real, 0, None, closed="left")], @@ -99,7 +98,7 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): internal={"pa1", "pa2"}, ) ], - "eta0": [Interval(Real, 0, None, closed="neither")], + "eta0": [Interval(Real, 0, None, closed="left")], "power_t": [Interval(Real, None, None, closed="neither")], "early_stopping": [bool], "validation_fraction": [Interval(Real, 0, 1, closed="neither")], @@ -186,19 +185,11 @@ def _get_loss_function(self, loss): raise ValueError("The loss %s is not supported. " % loss) from e def _get_learning_rate_type(self, learning_rate): - try: - return LEARNING_RATE_TYPES[learning_rate] - except KeyError as e: - raise ValueError( - "learning rate %s is not supported. " % learning_rate - ) from e + return LEARNING_RATE_TYPES[learning_rate] def _get_penalty_type(self, penalty): penalty = str(penalty).lower() - try: - return PENALTY_TYPES[penalty] - except KeyError as e: - raise ValueError("Penalty %s is not supported. " % penalty) from e + return PENALTY_TYPES[penalty] def _allocate_parameter_mem( self, n_classes, n_features, coef_init=None, intercept_init=None, one_class=0 @@ -358,7 +349,6 @@ def _prepare_fit_binary(est, y, i): return y_i, coef, intercept, average_coef, average_intercept -@validate_params() def fit_binary( est, i, @@ -1374,6 +1364,22 @@ class BaseSGDRegressor(RegressorMixin, BaseSGD): "squared_epsilon_insensitive": (SquaredEpsilonInsensitive, DEFAULT_EPSILON), } + _parameter_constraints = { + **BaseSGD._parameter_constraints, + "loss": [ + StrOptions( + { + "squared_error", + "squared_loss", + "huber", + "epsilon_insensitive", + "squared_epsilon_insensitive" + }, + deprecated={"squared_loss"}, + ) + ] + } + @abstractmethod def __init__( self, @@ -2131,6 +2137,18 @@ class SGDOneClassSVM(BaseSGD, OutlierMixin): loss_functions = {"hinge": (Hinge, 1.0)} + _parameter_constraints = { + **BaseSGD._parameter_constraints, + "loss": [ + StrOptions( + { + "hinge" + }, + ) + ], + "nu":[Interval(Integral,0,1,closed="right")] + } + def __init__( self, nu=0.5, @@ -2172,13 +2190,6 @@ def __init__( average=average, ) - def _validate_params(self, for_partial_fit=False): - """Validate input params.""" - if not (0 < self.nu <= 1): - raise ValueError("nu must be in (0, 1], got nu=%f" % self.nu) - - super(SGDOneClassSVM, self)._validate_params(for_partial_fit=for_partial_fit) - def _fit_one_class(self, X, alpha, C, sample_weight, learning_rate, max_iter): """Uses SGD implementation with X and y=np.ones(n_samples).""" @@ -2355,7 +2366,7 @@ def partial_fit(self, X, y=None, sample_weight=None): """ alpha = self.nu / 2 - self._validate_params(for_partial_fit=True) + self._revalidate_params(for_partial_fit=True) return self._partial_fit( X, @@ -2380,7 +2391,7 @@ def _fit( offset_init=None, sample_weight=None, ): - self._validate_params() + self._revalidate_params() if self.warm_start and hasattr(self, "coef_"): if coef_init is None: From 8e61778a6945c091d75363c48d6e9fc9dbdfaee6 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Sun, 5 Jun 2022 01:08:18 +0000 Subject: [PATCH 10/35] black lint --- sklearn/linear_model/_stochastic_gradient.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index c9c052b20547e..34acc4ecd68ed 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -1373,11 +1373,11 @@ class BaseSGDRegressor(RegressorMixin, BaseSGD): "squared_loss", "huber", "epsilon_insensitive", - "squared_epsilon_insensitive" + "squared_epsilon_insensitive", }, deprecated={"squared_loss"}, ) - ] + ], } @abstractmethod @@ -2141,12 +2141,10 @@ class SGDOneClassSVM(BaseSGD, OutlierMixin): **BaseSGD._parameter_constraints, "loss": [ StrOptions( - { - "hinge" - }, + {"hinge"}, ) ], - "nu":[Interval(Integral,0,1,closed="right")] + "nu": [Interval(Integral, 0, 1, closed="right")], } def __init__( From 76fcdf6760d55df60d2cffc423e02da38fb0d08d Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Mon, 6 Jun 2022 08:34:08 +0000 Subject: [PATCH 11/35] bug fix --- sklearn/linear_model/_stochastic_gradient.py | 33 +++++++++++++------- sklearn/tests/test_common.py | 3 -- sklearn/utils/estimator_checks.py | 8 +++++ 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 34acc4ecd68ed..ebfd9123380b2 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -83,7 +83,7 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "loss": [], "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None], "alpha": [Interval(Real, 0, None, closed="left")], - "C": [Interval(Integral, 1, None, closed="left")], + # "C": [Interval(Integral, 1, None, closed="left")], "l1_ratio": [Interval(Real, 0, 1, closed="both")], "fit_intercept": [bool], "max_iter": [Interval(Integral, 1, None, closed="left")], @@ -94,7 +94,7 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "random_state": ["random_state"], "learning_rate": [ StrOptions( - {"constant", "optimal", "invscaling", "adaptive"}, + {"constant", "optimal", "invscaling", "adaptive","pa1", "pa2"}, internal={"pa1", "pa2"}, ) ], @@ -1179,6 +1179,8 @@ class SGDClassifier(BaseSGDClassifier): [1] """ + _parameter_constraints = BaseSGDClassifier._parameter_constraints + def __init__( self, loss="hinge", @@ -2138,13 +2140,24 @@ class SGDOneClassSVM(BaseSGD, OutlierMixin): loss_functions = {"hinge": (Hinge, 1.0)} _parameter_constraints = { - **BaseSGD._parameter_constraints, - "loss": [ + # **BaseSGD._parameter_constraints, + "nu": [Interval(Real, 0.0, 1.0, closed="right")], + "fit_intercept": [bool], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "tol": [Interval(Real, 0, None, closed="left"), None], + "shuffle": [bool], + "verbose": [Interval(Integral, 0, None, closed="left")], + "random_state": ["random_state"], + "learning_rate": [ StrOptions( - {"hinge"}, + {"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}, + internal={"pa1", "pa2"}, ) ], - "nu": [Interval(Integral, 0, 1, closed="right")], + "eta0": [Interval(Real, 0, None, closed="left")], + "power_t": [Interval(Real, None, None, closed="neither")], + "warm_start": [bool], + "average": [Interval(Integral, 1, None, closed="left"), bool], } def __init__( @@ -2162,7 +2175,6 @@ def __init__( warm_start=False, average=False, ): - alpha = nu / 2 self.nu = nu super(SGDOneClassSVM, self).__init__( @@ -2363,9 +2375,8 @@ def partial_fit(self, X, y=None, sample_weight=None): Returns a fitted instance of self. """ - alpha = self.nu / 2 self._revalidate_params(for_partial_fit=True) - + alpha = self.nu / 2 return self._partial_fit( X, alpha, @@ -2389,8 +2400,6 @@ def _fit( offset_init=None, sample_weight=None, ): - self._revalidate_params() - if self.warm_start and hasattr(self, "coef_"): if coef_init is None: coef_init = self.coef_ @@ -2461,7 +2470,7 @@ def fit(self, X, y=None, coef_init=None, offset_init=None, sample_weight=None): self : object Returns a fitted instance of self. """ - + self._revalidate_params() alpha = self.nu / 2 self._fit( X, diff --git a/sklearn/tests/test_common.py b/sklearn/tests/test_common.py index 17ad851a3d7bd..2ea5cdbfa0fc0 100644 --- a/sklearn/tests/test_common.py +++ b/sklearn/tests/test_common.py @@ -601,9 +601,6 @@ def test_estimators_do_not_raise_errors_in_init_or_set_params(Estimator): "RidgeClassifier", "RidgeClassifierCV", "RobustScaler", - "SGDClassifier", - "SGDOneClassSVM", - "SGDRegressor", "SVC", "SVR", "SelectFdr", diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index c59f6eae7179a..c9ca39a1ddada 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -4055,6 +4055,13 @@ def check_param_validation(name, estimator_orig): err_msg = ( f"Mismatch between _parameter_constraints and the parameters of {name}." ) + for hh in estimator_orig._parameter_constraints.keys(): + if not hh in estimator_params: + print(f"{hh} not in found in {name} constructor") + for hh in estimator_params: + if not hh in estimator_orig._parameter_constraints.keys(): + print(f"{hh} not found in {name} _parameter_constraints") + print(estimator_orig._parameter_constraints.keys()," != ",estimator_params) assert estimator_orig._parameter_constraints.keys() == estimator_params, err_msg # this object does not have a valid type for sure for all params @@ -4076,6 +4083,7 @@ def check_param_validation(name, estimator_orig): estimator.set_params(**{param_name: param_with_bad_type}) for method in methods: + print(f"{method} ,{X}, {y}, {estimator}") with raises(ValueError, match=match, err_msg=err_msg): getattr(estimator, method)(X, y) From 8fabf2f2362eb6e9db28e447cdd534480f6853cf Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Mon, 6 Jun 2022 08:37:24 +0000 Subject: [PATCH 12/35] fix --- sklearn/utils/estimator_checks.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index c9ca39a1ddada..c59f6eae7179a 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -4055,13 +4055,6 @@ def check_param_validation(name, estimator_orig): err_msg = ( f"Mismatch between _parameter_constraints and the parameters of {name}." ) - for hh in estimator_orig._parameter_constraints.keys(): - if not hh in estimator_params: - print(f"{hh} not in found in {name} constructor") - for hh in estimator_params: - if not hh in estimator_orig._parameter_constraints.keys(): - print(f"{hh} not found in {name} _parameter_constraints") - print(estimator_orig._parameter_constraints.keys()," != ",estimator_params) assert estimator_orig._parameter_constraints.keys() == estimator_params, err_msg # this object does not have a valid type for sure for all params @@ -4083,7 +4076,6 @@ def check_param_validation(name, estimator_orig): estimator.set_params(**{param_name: param_with_bad_type}) for method in methods: - print(f"{method} ,{X}, {y}, {estimator}") with raises(ValueError, match=match, err_msg=err_msg): getattr(estimator, method)(X, y) From 93ee1a8cceb0cf20ba22d8fc3e5e0a4eedd034bc Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Mon, 6 Jun 2022 09:33:21 +0000 Subject: [PATCH 13/35] fix perceptron --- sklearn/linear_model/_perceptron.py | 23 ++++++++++++++++++++++- sklearn/tests/test_common.py | 1 - sklearn/utils/estimator_checks.py | 7 +++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/sklearn/linear_model/_perceptron.py b/sklearn/linear_model/_perceptron.py index 2e2eb963cc551..42ceae4840164 100644 --- a/sklearn/linear_model/_perceptron.py +++ b/sklearn/linear_model/_perceptron.py @@ -1,7 +1,9 @@ # Author: Mathieu Blondel # License: BSD 3 clause +from numbers import Integral, Real -from ._stochastic_gradient import BaseSGDClassifier +from ._stochastic_gradient import BaseSGDClassifier, BaseSGD +from ..utils._param_validation import StrOptions, Interval class Perceptron(BaseSGDClassifier): @@ -164,6 +166,25 @@ class Perceptron(BaseSGDClassifier): 0.939... """ + _parameter_constraints = { + "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None], + "alpha": [Interval(Real, 0, None, closed="left")], + "l1_ratio": [Interval(Real, 0, 1, closed="both")], + "fit_intercept": [bool], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "tol": [Interval(Real, 0, None, closed="left"), None], + "shuffle": [bool], + "verbose": [Interval(Integral, 0, None, closed="left")], + "eta0": [Interval(Real, 0, None, closed="left")], + "random_state": ["random_state"], + "early_stopping": [bool], + "validation_fraction": [Interval(Real, 0, 1, closed="neither")], + "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], + "warm_start": [bool], + "n_jobs": [None, Integral], + "class_weight": [StrOptions({"balanced"}), None], # a bit tricky? + } + def __init__( self, *, diff --git a/sklearn/tests/test_common.py b/sklearn/tests/test_common.py index 2ea5cdbfa0fc0..cfb66d1218ae5 100644 --- a/sklearn/tests/test_common.py +++ b/sklearn/tests/test_common.py @@ -577,7 +577,6 @@ def test_estimators_do_not_raise_errors_in_init_or_set_params(Estimator): "PassiveAggressiveClassifier", "PassiveAggressiveRegressor", "PatchExtractor", - "Perceptron", "PoissonRegressor", "PolynomialCountSketch", "PolynomialFeatures", diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index c59f6eae7179a..840aba8b3fc59 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -4055,6 +4055,13 @@ def check_param_validation(name, estimator_orig): err_msg = ( f"Mismatch between _parameter_constraints and the parameters of {name}." ) + for hh in estimator_orig._parameter_constraints.keys(): + if not hh in estimator_params: + print(f"{hh} not in found in {name} constructor") + for hh in estimator_params: + if not hh in estimator_orig._parameter_constraints.keys(): + print(f"{hh} not found in {name} _parameter_constraints") + print(estimator_orig._parameter_constraints.keys()," != ",estimator_params) assert estimator_orig._parameter_constraints.keys() == estimator_params, err_msg # this object does not have a valid type for sure for all params From 1c17f73aa4ee3488b79d3a49811c1d964f1fcba0 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Mon, 6 Jun 2022 09:37:05 +0000 Subject: [PATCH 14/35] black lint --- sklearn/linear_model/_stochastic_gradient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index ebfd9123380b2..a2239be0970e5 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -94,7 +94,7 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "random_state": ["random_state"], "learning_rate": [ StrOptions( - {"constant", "optimal", "invscaling", "adaptive","pa1", "pa2"}, + {"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}, internal={"pa1", "pa2"}, ) ], From 302a3298ebc6e6b28f685a39bfabcb1f5d2a44b9 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Mon, 6 Jun 2022 10:52:45 +0000 Subject: [PATCH 15/35] fix tests --- sklearn/linear_model/_stochastic_gradient.py | 29 ++++++++++++++++---- sklearn/linear_model/tests/test_sgd.py | 17 +----------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index a2239be0970e5..30fe1dfa43cd5 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -87,7 +87,7 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "l1_ratio": [Interval(Real, 0, 1, closed="both")], "fit_intercept": [bool], "max_iter": [Interval(Integral, 1, None, closed="left")], - "tol": [Interval(Real, 0, None, closed="left"), None], + "tol": [Interval(Real, None, None, closed="neither"), None], "shuffle": [bool], "verbose": [Interval(Integral, 0, None, closed="left")], "epsilon": [Interval(Real, 0, None, closed="left")], @@ -104,7 +104,7 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "validation_fraction": [Interval(Real, 0, 1, closed="neither")], "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], "warm_start": [bool], - "average": [Interval(Integral, 1, None, closed="left"), bool], + "average": [Interval(Integral, 0, None, closed="left"), bool], } def __init__( @@ -162,6 +162,9 @@ def _revalidate_params(self, for_partial_fit=False): if self.early_stopping and for_partial_fit: raise ValueError("early_stopping should be False with partial_fit") + if self.learning_rate in ("constant", "invscaling", "adaptive"): + if self.eta0 <= 0.0: + raise ValueError("eta0 must be > 0") if self.learning_rate == "optimal" and self.alpha == 0: raise ValueError( "alpha must be > 0 since " @@ -173,6 +176,22 @@ def _revalidate_params(self, for_partial_fit=False): self._get_penalty_type(self.penalty) self._get_learning_rate_type(self.learning_rate) + # TODO(1.2): remove "squared_loss" + if self.loss == "squared_loss": + warnings.warn( + "The loss 'squared_loss' was deprecated in v1.0 and will be " + "removed in version 1.2. Use `loss='squared_error'` which is " + "equivalent.", + FutureWarning, + ) + # TODO(1.3): remove "log" + if self.loss == "log": + warnings.warn( + "The loss 'log' was deprecated in v1.1 and will be removed in version " + "1.3. Use `loss='log_loss'` which is equivalent.", + FutureWarning, + ) + def _get_loss_function(self, loss): """Get concrete ``LossFunction`` object for str ``loss``.""" try: @@ -521,7 +540,7 @@ class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta): ) ], "n_jobs": [None, Integral], - "class_weight": [StrOptions({"balanced"}), None], # a bit tricky? + "class_weight": [StrOptions({"balanced"}), dict, None], # a bit tricky? } @abstractmethod @@ -2144,7 +2163,7 @@ class SGDOneClassSVM(BaseSGD, OutlierMixin): "nu": [Interval(Real, 0.0, 1.0, closed="right")], "fit_intercept": [bool], "max_iter": [Interval(Integral, 1, None, closed="left")], - "tol": [Interval(Real, 0, None, closed="left"), None], + "tol": [Interval(Real, None, None, closed="left"), None], "shuffle": [bool], "verbose": [Interval(Integral, 0, None, closed="left")], "random_state": ["random_state"], @@ -2157,7 +2176,7 @@ class SGDOneClassSVM(BaseSGD, OutlierMixin): "eta0": [Interval(Real, 0, None, closed="left")], "power_t": [Interval(Real, None, None, closed="neither")], "warm_start": [bool], - "average": [Interval(Integral, 1, None, closed="left"), bool], + "average": [Interval(Integral, 0, None, closed="left"), bool], } def __init__( diff --git a/sklearn/linear_model/tests/test_sgd.py b/sklearn/linear_model/tests/test_sgd.py index 1a48afeeb48db..419fb7ba70ed0 100644 --- a/sklearn/linear_model/tests/test_sgd.py +++ b/sklearn/linear_model/tests/test_sgd.py @@ -231,23 +231,8 @@ def asgd(klass, X, y, eta, alpha, weight_init=None, intercept_init=0.0): @pytest.mark.parametrize( "params, err_msg", [ - ({"alpha": -0.1}, "alpha must be >= 0"), - ({"penalty": "foobar", "l1_ratio": 0.85}, "Penalty foobar is not supported"), - ({"loss": "foobar"}, "The loss foobar is not supported"), - ({"l1_ratio": 1.1}, r"l1_ratio must be in \[0, 1\]"), - ({"learning_rate": ""}, "learning rate is not supported"), - ({"nu": -0.5}, r"nu must be in \(0, 1]"), - ({"nu": 2}, r"nu must be in \(0, 1]"), ({"alpha": 0, "learning_rate": "optimal"}, "alpha must be > 0"), - ({"eta0": 0, "learning_rate": "constant"}, "eta0 must be > 0"), - ({"max_iter": -1}, "max_iter must be > zero"), - ({"shuffle": "false"}, "shuffle must be either True or False"), - ({"early_stopping": "false"}, "early_stopping must be either True or False"), - ( - {"validation_fraction": -0.1}, - r"validation_fraction must be in range \(0, 1\)", - ), - ({"n_iter_no_change": 0}, "n_iter_no_change must be >= 1"), + ({"eta0": 0, "learning_rate": "constant"}, "eta0 must be > 0") ], # Avoid long error messages in test names: # https://github.com/scikit-learn/scikit-learn/issues/21362 From b85ffa42da5154582d7e1bcc398ffea98e856d5e Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Mon, 6 Jun 2022 10:54:24 +0000 Subject: [PATCH 16/35] black lint --- sklearn/linear_model/tests/test_sgd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/linear_model/tests/test_sgd.py b/sklearn/linear_model/tests/test_sgd.py index 419fb7ba70ed0..0e9a3250c3748 100644 --- a/sklearn/linear_model/tests/test_sgd.py +++ b/sklearn/linear_model/tests/test_sgd.py @@ -232,7 +232,7 @@ def asgd(klass, X, y, eta, alpha, weight_init=None, intercept_init=0.0): "params, err_msg", [ ({"alpha": 0, "learning_rate": "optimal"}, "alpha must be > 0"), - ({"eta0": 0, "learning_rate": "constant"}, "eta0 must be > 0") + ({"eta0": 0, "learning_rate": "constant"}, "eta0 must be > 0"), ], # Avoid long error messages in test names: # https://github.com/scikit-learn/scikit-learn/issues/21362 From 45ef9a25992db33ef31d3baf9e0fdbb356417560 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Mon, 6 Jun 2022 12:31:08 +0000 Subject: [PATCH 17/35] optimize imports --- sklearn/linear_model/_stochastic_gradient.py | 40 ++++++++++---------- sklearn/linear_model/tests/test_sgd.py | 28 +++++++------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 30fe1dfa43cd5..4888aa0c2e970 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -6,39 +6,37 @@ Descent (SGD). """ -import numpy as np -from numbers import Integral, Real import warnings - from abc import ABCMeta, abstractmethod +from numbers import Integral, Real +import numpy as np from joblib import Parallel -from ..base import clone, is_classifier from ._base import LinearClassifierMixin, SparseCoefMixin from ._base import make_dataset +from ._sgd_fast import EpsilonInsensitive +from ._sgd_fast import Hinge +from ._sgd_fast import Huber +from ._sgd_fast import Log +from ._sgd_fast import ModifiedHuber +from ._sgd_fast import SquaredEpsilonInsensitive +from ._sgd_fast import SquaredHinge +from ._sgd_fast import SquaredLoss +from ._sgd_fast import _plain_sgd from ..base import BaseEstimator, RegressorMixin, OutlierMixin +from ..base import clone, is_classifier +from ..exceptions import ConvergenceWarning +from ..model_selection import StratifiedShuffleSplit, ShuffleSplit from ..utils import check_random_state -from ..utils.metaestimators import available_if -from ..utils.extmath import safe_sparse_dot -from ..utils.multiclass import _check_partial_fit_first_call -from ..utils.validation import check_is_fitted, _check_sample_weight +from ..utils import compute_class_weight from ..utils._param_validation import Interval from ..utils._param_validation import StrOptions +from ..utils.extmath import safe_sparse_dot from ..utils.fixes import delayed -from ..exceptions import ConvergenceWarning -from ..model_selection import StratifiedShuffleSplit, ShuffleSplit - -from ._sgd_fast import _plain_sgd -from ..utils import compute_class_weight -from ._sgd_fast import Hinge -from ._sgd_fast import SquaredHinge -from ._sgd_fast import Log -from ._sgd_fast import ModifiedHuber -from ._sgd_fast import SquaredLoss -from ._sgd_fast import Huber -from ._sgd_fast import EpsilonInsensitive -from ._sgd_fast import SquaredEpsilonInsensitive +from ..utils.metaestimators import available_if +from ..utils.multiclass import _check_partial_fit_first_call +from ..utils.validation import check_is_fitted, _check_sample_weight LEARNING_RATE_TYPES = { "constant": 1, diff --git a/sklearn/linear_model/tests/test_sgd.py b/sklearn/linear_model/tests/test_sgd.py index 0e9a3250c3748..a2c956c6cb32b 100644 --- a/sklearn/linear_model/tests/test_sgd.py +++ b/sklearn/linear_model/tests/test_sgd.py @@ -1,29 +1,28 @@ import pickle +from unittest.mock import Mock import joblib -import pytest import numpy as np +import pytest import scipy.sparse as sp -from unittest.mock import Mock - -from sklearn.utils._testing import assert_allclose -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import ignore_warnings from sklearn import linear_model, datasets, metrics from sklearn.base import clone, is_classifier -from sklearn.svm import OneClassSVM -from sklearn.preprocessing import LabelEncoder, scale, MinMaxScaler -from sklearn.preprocessing import StandardScaler -from sklearn.kernel_approximation import Nystroem -from sklearn.pipeline import make_pipeline from sklearn.exceptions import ConvergenceWarning -from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit +from sklearn.kernel_approximation import Nystroem from sklearn.linear_model import _sgd_fast as sgd_fast from sklearn.linear_model import _stochastic_gradient from sklearn.model_selection import RandomizedSearchCV +from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import LabelEncoder, scale, MinMaxScaler +from sklearn.preprocessing import StandardScaler +from sklearn.svm import OneClassSVM +from sklearn.utils._testing import assert_allclose +from sklearn.utils._testing import assert_almost_equal +from sklearn.utils._testing import assert_array_almost_equal +from sklearn.utils._testing import assert_array_equal +from sklearn.utils._testing import ignore_warnings def _update_kwargs(kwargs): @@ -2121,7 +2120,6 @@ def test_SGDClassifier_fit_for_all_backends(backend): ], ) def test_loss_deprecated(old_loss, new_loss, Estimator): - # Note: class BaseSGD calls self._validate_params() in __init__, therefore # even instantiation of class raises FutureWarning for deprecated losses. with pytest.warns(FutureWarning, match=f"The loss '{old_loss}' was deprecated"): From 419bcd7fcceb7696eea6051d059f31536417b6dd Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Mon, 6 Jun 2022 16:16:17 +0000 Subject: [PATCH 18/35] test fix --- sklearn/linear_model/_passive_aggressive.py | 43 +++++++++++++++++-- sklearn/linear_model/_stochastic_gradient.py | 7 ++- .../tests/test_passive_aggressive.py | 32 -------------- sklearn/utils/_param_validation.py | 10 ++++- 4 files changed, 53 insertions(+), 39 deletions(-) diff --git a/sklearn/linear_model/_passive_aggressive.py b/sklearn/linear_model/_passive_aggressive.py index 65f754ba35f55..752f98dfa2cf8 100644 --- a/sklearn/linear_model/_passive_aggressive.py +++ b/sklearn/linear_model/_passive_aggressive.py @@ -1,9 +1,11 @@ # Authors: Rob Zinkov, Mathieu Blondel # License: BSD 3 clause +from numbers import Integral, Real from ._stochastic_gradient import BaseSGDClassifier from ._stochastic_gradient import BaseSGDRegressor from ._stochastic_gradient import DEFAULT_EPSILON +from ..utils._param_validation import Interval, StrOptions class PassiveAggressiveClassifier(BaseSGDClassifier): @@ -172,6 +174,24 @@ class PassiveAggressiveClassifier(BaseSGDClassifier): [1] """ + _parameter_constraints = { + "loss": [StrOptions({"hinge", "squared_hinge"})], + "C": [Interval(Real , None, None, closed="neither")], + "fit_intercept": [bool], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "tol": [Interval(Real, None, None, closed="neither"), None], + "shuffle": [bool], + "verbose": [Interval(Integral, 0, None, closed="left")], + "random_state": ["random_state"], + "early_stopping": [bool], + "validation_fraction": [Interval(Real, 0, 1, closed="neither")], + "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], + "warm_start": [bool], + "average": [Interval(Integral, 0, None, closed="left"), bool], + "n_jobs": [None, Integral], + "class_weight": [StrOptions({"balanced"}), dict, None], + } + def __init__( self, *, @@ -236,7 +256,7 @@ def partial_fit(self, X, y, classes=None): self : object Fitted estimator. """ - self._validate_params(for_partial_fit=True) + self._revalidate_params(for_partial_fit=True) if self.class_weight == "balanced": raise ValueError( "class_weight 'balanced' is not supported for " @@ -286,7 +306,7 @@ def fit(self, X, y, coef_init=None, intercept_init=None): self : object Fitted estimator. """ - self._validate_params() + self._revalidate_params() lr = "pa1" if self.loss == "hinge" else "pa2" return self._fit( X, @@ -444,6 +464,23 @@ class PassiveAggressiveRegressor(BaseSGDRegressor): >>> print(regr.predict([[0, 0, 0, 0]])) [-0.02306214] """ + _parameter_constraints = { + "loss": [StrOptions({"epsilon_insensitive","squared_epsilon_insensitive"})], + "C": [Interval(Real, None, None, closed="neither")], + "fit_intercept": [bool], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "tol": [Interval(Real, None, None, closed="neither"), None], + "shuffle": [bool], + "verbose": [Interval(Integral, 0, None, closed="left")], + "random_state": ["random_state"], + "early_stopping": [bool], + "validation_fraction": [Interval(Real, 0, 1, closed="neither")], + "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], + "warm_start": [bool], + "average": [Interval(Integral, 0, None, closed="left"), bool], + "epsilon": [Interval(Real, 0, None, closed="left")], + + } def __init__( self, @@ -499,7 +536,7 @@ def partial_fit(self, X, y): self : object Fitted estimator. """ - self._validate_params(for_partial_fit=True) + self._revalidate_params(for_partial_fit=True) lr = "pa1" if self.loss == "epsilon_insensitive" else "pa2" return self._partial_fit( X, diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 4888aa0c2e970..4dfac346567a3 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -78,10 +78,9 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): """Base class for SGD classification and regression.""" _parameter_constraints = { - "loss": [], "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None], "alpha": [Interval(Real, 0, None, closed="left")], - # "C": [Interval(Integral, 1, None, closed="left")], + # "C": [Interval(Real , 1, None, closed="left")], "l1_ratio": [Interval(Real, 0, 1, closed="both")], "fit_intercept": [bool], "max_iter": [Interval(Integral, 1, None, closed="left")], @@ -928,6 +927,10 @@ def fit(self, X, y, coef_init=None, intercept_init=None, sample_weight=None): sample_weight=sample_weight, ) + @property + def parameter_constraints(self): + return self._parameter_constraints + class SGDClassifier(BaseSGDClassifier): """Linear classifiers (SVM, logistic regression, etc.) with SGD training. diff --git a/sklearn/linear_model/tests/test_passive_aggressive.py b/sklearn/linear_model/tests/test_passive_aggressive.py index 3ff92bd69a43b..54fbecdccd81a 100644 --- a/sklearn/linear_model/tests/test_passive_aggressive.py +++ b/sklearn/linear_model/tests/test_passive_aggressive.py @@ -284,35 +284,3 @@ def test_regressor_undefined_methods(): reg = PassiveAggressiveRegressor(max_iter=100) with pytest.raises(AttributeError): reg.transform(X) - - -@pytest.mark.parametrize( - "klass", [PassiveAggressiveClassifier, PassiveAggressiveRegressor] -) -@pytest.mark.parametrize("fit_method", ["fit", "partial_fit"]) -@pytest.mark.parametrize( - "params, err_msg", - [ - ({"loss": "foobar"}, "The loss foobar is not supported"), - ({"max_iter": -1}, "max_iter must be > zero"), - ({"shuffle": "false"}, "shuffle must be either True or False"), - ({"early_stopping": "false"}, "early_stopping must be either True or False"), - ( - {"validation_fraction": -0.1}, - r"validation_fraction must be in range \(0, 1\)", - ), - ({"n_iter_no_change": 0}, "n_iter_no_change must be >= 1"), - ], -) -def test_passive_aggressive_estimator_params_validation( - klass, fit_method, params, err_msg -): - """Validate parameters in the different PassiveAggressive estimators.""" - sgd_estimator = klass(**params) - - with pytest.raises(ValueError, match=err_msg): - if is_classifier(sgd_estimator) and fit_method == "partial_fit": - fit_params = {"classes": np.unique(y)} - else: - fit_params = {} - getattr(sgd_estimator, fit_method)(X, y, **fit_params) diff --git a/sklearn/utils/_param_validation.py b/sklearn/utils/_param_validation.py index 91632c3f95907..44f6cda40b14f 100644 --- a/sklearn/utils/_param_validation.py +++ b/sklearn/utils/_param_validation.py @@ -36,6 +36,13 @@ def validate_parameter_constraints(parameter_constraints, params, caller_name): caller_name : str The name of the estimator or function or method that called this function. """ + for hh in params.keys(): + if not hh in parameter_constraints.keys(): + print(f"{hh} not found in _parameter_constraints") + for hh in parameter_constraints.keys(): + if not hh in params.keys(): + print(f"{hh} not found in constructor") + if params.keys() != parameter_constraints.keys(): raise ValueError( f"The parameter constraints {list(parameter_constraints.keys())} do not " @@ -119,7 +126,6 @@ def validate_params(parameter_constraints): def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): - func_sig = signature(func) # Map *args/**kwargs to the function signature @@ -400,7 +406,7 @@ class _RandomStates(_Constraint): def __init__(self): self._constraints = [ - Interval(Integral, 0, 2**32 - 1, closed="both"), + Interval(Integral, 0, 2 ** 32 - 1, closed="both"), _InstancesOf(np.random.RandomState), _NoneConstraint(), ] From e29f5133376622d6e49779d45250ef054d05c727 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Mon, 6 Jun 2022 16:16:58 +0000 Subject: [PATCH 19/35] black lint --- sklearn/linear_model/_passive_aggressive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sklearn/linear_model/_passive_aggressive.py b/sklearn/linear_model/_passive_aggressive.py index 752f98dfa2cf8..6972985c8c300 100644 --- a/sklearn/linear_model/_passive_aggressive.py +++ b/sklearn/linear_model/_passive_aggressive.py @@ -176,7 +176,7 @@ class PassiveAggressiveClassifier(BaseSGDClassifier): _parameter_constraints = { "loss": [StrOptions({"hinge", "squared_hinge"})], - "C": [Interval(Real , None, None, closed="neither")], + "C": [Interval(Real, None, None, closed="neither")], "fit_intercept": [bool], "max_iter": [Interval(Integral, 1, None, closed="left")], "tol": [Interval(Real, None, None, closed="neither"), None], @@ -464,8 +464,9 @@ class PassiveAggressiveRegressor(BaseSGDRegressor): >>> print(regr.predict([[0, 0, 0, 0]])) [-0.02306214] """ + _parameter_constraints = { - "loss": [StrOptions({"epsilon_insensitive","squared_epsilon_insensitive"})], + "loss": [StrOptions({"epsilon_insensitive", "squared_epsilon_insensitive"})], "C": [Interval(Real, None, None, closed="neither")], "fit_intercept": [bool], "max_iter": [Interval(Integral, 1, None, closed="left")], @@ -479,7 +480,6 @@ class PassiveAggressiveRegressor(BaseSGDRegressor): "warm_start": [bool], "average": [Interval(Integral, 0, None, closed="left"), bool], "epsilon": [Interval(Real, 0, None, closed="left")], - } def __init__( From 5fe1785c69368962f32dee971aa9fc9523632e32 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Mon, 6 Jun 2022 16:25:03 +0000 Subject: [PATCH 20/35] clean up --- sklearn/utils/_param_validation.py | 9 +-------- sklearn/utils/estimator_checks.py | 8 +------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/sklearn/utils/_param_validation.py b/sklearn/utils/_param_validation.py index 44f6cda40b14f..17ad909a1a690 100644 --- a/sklearn/utils/_param_validation.py +++ b/sklearn/utils/_param_validation.py @@ -36,13 +36,6 @@ def validate_parameter_constraints(parameter_constraints, params, caller_name): caller_name : str The name of the estimator or function or method that called this function. """ - for hh in params.keys(): - if not hh in parameter_constraints.keys(): - print(f"{hh} not found in _parameter_constraints") - for hh in parameter_constraints.keys(): - if not hh in params.keys(): - print(f"{hh} not found in constructor") - if params.keys() != parameter_constraints.keys(): raise ValueError( f"The parameter constraints {list(parameter_constraints.keys())} do not " @@ -406,7 +399,7 @@ class _RandomStates(_Constraint): def __init__(self): self._constraints = [ - Interval(Integral, 0, 2 ** 32 - 1, closed="both"), + Interval(Integral, 0, 2**32 - 1, closed="both"), _InstancesOf(np.random.RandomState), _NoneConstraint(), ] diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index 840aba8b3fc59..01edffb2fac25 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -4055,13 +4055,7 @@ def check_param_validation(name, estimator_orig): err_msg = ( f"Mismatch between _parameter_constraints and the parameters of {name}." ) - for hh in estimator_orig._parameter_constraints.keys(): - if not hh in estimator_params: - print(f"{hh} not in found in {name} constructor") - for hh in estimator_params: - if not hh in estimator_orig._parameter_constraints.keys(): - print(f"{hh} not found in {name} _parameter_constraints") - print(estimator_orig._parameter_constraints.keys()," != ",estimator_params) + assert estimator_orig._parameter_constraints.keys() == estimator_params, err_msg # this object does not have a valid type for sure for all params From e4dc9c2bd149fa0e3618f79c45b4279f385c9649 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Mon, 6 Jun 2022 16:30:13 +0000 Subject: [PATCH 21/35] clean up --- sklearn/linear_model/_perceptron.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/linear_model/_perceptron.py b/sklearn/linear_model/_perceptron.py index 42ceae4840164..ec8e0c285d38e 100644 --- a/sklearn/linear_model/_perceptron.py +++ b/sklearn/linear_model/_perceptron.py @@ -2,7 +2,7 @@ # License: BSD 3 clause from numbers import Integral, Real -from ._stochastic_gradient import BaseSGDClassifier, BaseSGD +from ._stochastic_gradient import BaseSGDClassifier from ..utils._param_validation import StrOptions, Interval From 1b538d1d327315a8e208b3d9b6f80389669b1a6d Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Mon, 6 Jun 2022 16:35:04 +0000 Subject: [PATCH 22/35] clean up --- sklearn/linear_model/tests/test_passive_aggressive.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sklearn/linear_model/tests/test_passive_aggressive.py b/sklearn/linear_model/tests/test_passive_aggressive.py index 54fbecdccd81a..72c2c31500611 100644 --- a/sklearn/linear_model/tests/test_passive_aggressive.py +++ b/sklearn/linear_model/tests/test_passive_aggressive.py @@ -3,7 +3,6 @@ import pytest -from sklearn.base import is_classifier from sklearn.utils._testing import assert_array_almost_equal from sklearn.utils._testing import assert_array_equal from sklearn.utils._testing import assert_almost_equal From ed78625982c3f3d37470ab1bee4a8b6724cd4da2 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Tue, 14 Jun 2022 03:36:12 +0000 Subject: [PATCH 23/35] build fix --- sklearn/linear_model/_stochastic_gradient.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 528c51a93c315..17d8c1ab3d383 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -91,8 +91,7 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "random_state": ["random_state"], "learning_rate": [ StrOptions( - {"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}, - internal={"pa1", "pa2"}, + {"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"} ) ], "eta0": [Interval(Real, 0, None, closed="left")], From 68509d43f270313c439def1ddc0df68c7d0f54a7 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Tue, 14 Jun 2022 03:40:17 +0000 Subject: [PATCH 24/35] build fix --- sklearn/linear_model/_stochastic_gradient.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 17d8c1ab3d383..1136286078c1c 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -2169,8 +2169,7 @@ class SGDOneClassSVM(BaseSGD, OutlierMixin): "random_state": ["random_state"], "learning_rate": [ StrOptions( - {"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}, - internal={"pa1", "pa2"}, + {"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"} ) ], "eta0": [Interval(Real, 0, None, closed="left")], From 39182062cf9b77fd2cbb882a1640935e3f61b59b Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Tue, 14 Jun 2022 03:46:55 +0000 Subject: [PATCH 25/35] black linting --- sklearn/linear_model/_stochastic_gradient.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 1136286078c1c..de5910adcd1d8 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -90,9 +90,7 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "epsilon": [Interval(Real, 0, None, closed="left")], "random_state": ["random_state"], "learning_rate": [ - StrOptions( - {"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"} - ) + StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}) ], "eta0": [Interval(Real, 0, None, closed="left")], "power_t": [Interval(Real, None, None, closed="neither")], @@ -2168,9 +2166,7 @@ class SGDOneClassSVM(BaseSGD, OutlierMixin): "verbose": [Interval(Integral, 0, None, closed="left")], "random_state": ["random_state"], "learning_rate": [ - StrOptions( - {"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"} - ) + StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}) ], "eta0": [Interval(Real, 0, None, closed="left")], "power_t": [Interval(Real, None, None, closed="neither")], From f1f9435100f2db902b31865ad5cae731b056405e Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Tue, 14 Jun 2022 11:08:16 +0000 Subject: [PATCH 26/35] fix bug --- sklearn/linear_model/_stochastic_gradient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index de5910adcd1d8..4eb4c3ec45f7f 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -84,7 +84,7 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): "l1_ratio": [Interval(Real, 0, 1, closed="both")], "fit_intercept": [bool], "max_iter": [Interval(Integral, 1, None, closed="left")], - "tol": [Interval(Real, None, None, closed="neither"), None], + "tol": [Interval(Real, None, None, closed="both"), None], "shuffle": [bool], "verbose": [Interval(Integral, 0, None, closed="left")], "epsilon": [Interval(Real, 0, None, closed="left")], From 8efc7bc9e7cbefccfa78fbf51541b4fd0f42f5a0 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Tue, 14 Jun 2022 11:36:37 +0000 Subject: [PATCH 27/35] fix bug --- sklearn/linear_model/_perceptron.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/linear_model/_perceptron.py b/sklearn/linear_model/_perceptron.py index ec8e0c285d38e..6b13d80a12b9c 100644 --- a/sklearn/linear_model/_perceptron.py +++ b/sklearn/linear_model/_perceptron.py @@ -182,7 +182,7 @@ class Perceptron(BaseSGDClassifier): "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], "warm_start": [bool], "n_jobs": [None, Integral], - "class_weight": [StrOptions({"balanced"}), None], # a bit tricky? + "class_weight": [StrOptions({"balanced"}), dict, None], # a bit tricky? } def __init__( From e58dcd80c82c4a98262601a50251b9143b6d4ae6 Mon Sep 17 00:00:00 2001 From: "Nwanna.Joseph" Date: Tue, 14 Jun 2022 14:48:49 +0000 Subject: [PATCH 28/35] fix bug --- sklearn/linear_model/_stochastic_gradient.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 4eb4c3ec45f7f..aade8f3bf3799 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -924,10 +924,6 @@ def fit(self, X, y, coef_init=None, intercept_init=None, sample_weight=None): sample_weight=sample_weight, ) - @property - def parameter_constraints(self): - return self._parameter_constraints - class SGDClassifier(BaseSGDClassifier): """Linear classifiers (SVM, logistic regression, etc.) with SGD training. From b8b7335f3e238676644360e971dc3a75c931772c Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 24 Jun 2022 10:35:30 +0200 Subject: [PATCH 29/35] fixes --- sklearn/linear_model/_stochastic_gradient.py | 69 ++++++++++---------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index aade8f3bf3799..e2dbaab3db687 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -6,37 +6,39 @@ Descent (SGD). """ +import numpy as np import warnings + from abc import ABCMeta, abstractmethod from numbers import Integral, Real -import numpy as np from joblib import Parallel +from ..base import clone, is_classifier from ._base import LinearClassifierMixin, SparseCoefMixin from ._base import make_dataset -from ._sgd_fast import EpsilonInsensitive -from ._sgd_fast import Hinge -from ._sgd_fast import Huber -from ._sgd_fast import Log -from ._sgd_fast import ModifiedHuber -from ._sgd_fast import SquaredEpsilonInsensitive -from ._sgd_fast import SquaredHinge -from ._sgd_fast import SquaredLoss -from ._sgd_fast import _plain_sgd from ..base import BaseEstimator, RegressorMixin, OutlierMixin -from ..base import clone, is_classifier -from ..exceptions import ConvergenceWarning -from ..model_selection import StratifiedShuffleSplit, ShuffleSplit from ..utils import check_random_state -from ..utils import compute_class_weight -from ..utils._param_validation import Interval -from ..utils._param_validation import StrOptions -from ..utils.extmath import safe_sparse_dot -from ..utils.fixes import delayed from ..utils.metaestimators import available_if +from ..utils.extmath import safe_sparse_dot from ..utils.multiclass import _check_partial_fit_first_call from ..utils.validation import check_is_fitted, _check_sample_weight +from ..utils._param_validation import Interval +from ..utils._param_validation import StrOptions +from ..utils.fixes import delayed +from ..exceptions import ConvergenceWarning +from ..model_selection import StratifiedShuffleSplit, ShuffleSplit + +from ._sgd_fast import _plain_sgd +from ..utils import compute_class_weight +from ._sgd_fast import Hinge +from ._sgd_fast import SquaredHinge +from ._sgd_fast import Log +from ._sgd_fast import ModifiedHuber +from ._sgd_fast import SquaredLoss +from ._sgd_fast import Huber +from ._sgd_fast import EpsilonInsensitive +from ._sgd_fast import SquaredEpsilonInsensitive LEARNING_RATE_TYPES = { "constant": 1, @@ -78,15 +80,15 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): """Base class for SGD classification and regression.""" _parameter_constraints = { - "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None], + "penalty": [StrOptions({"l2", "l1", "elasticnet"})], "alpha": [Interval(Real, 0, None, closed="left")], - # "C": [Interval(Real , 1, None, closed="left")], + # "C": [Interval(Real , 0, None, closed="right")], "l1_ratio": [Interval(Real, 0, 1, closed="both")], - "fit_intercept": [bool], + "fit_intercept": ["boolean"], "max_iter": [Interval(Integral, 1, None, closed="left")], - "tol": [Interval(Real, None, None, closed="both"), None], - "shuffle": [bool], - "verbose": [Interval(Integral, 0, None, closed="left")], + "tol": [Interval(Real, 0, None, closed="left")], + "shuffle": ["boolean"], + "verbose": ["verbose"], "epsilon": [Interval(Real, 0, None, closed="left")], "random_state": ["random_state"], "learning_rate": [ @@ -94,11 +96,11 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): ], "eta0": [Interval(Real, 0, None, closed="left")], "power_t": [Interval(Real, None, None, closed="neither")], - "early_stopping": [bool], + "early_stopping": ["boolean"], "validation_fraction": [Interval(Real, 0, 1, closed="neither")], "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], - "warm_start": [bool], - "average": [Interval(Integral, 0, None, closed="left"), bool], + "warm_start": ["boolean"], + "average": [Interval(Integral, 1, None, closed="left"), bool, np.bool_], } def __init__( @@ -152,8 +154,6 @@ def fit(self, X, y): def _revalidate_params(self, for_partial_fit=False): """Validate input params.""" - self._validate_params() - if self.early_stopping and for_partial_fit: raise ValueError("early_stopping should be False with partial_fit") if self.learning_rate in ("constant", "invscaling", "adaptive"): @@ -533,8 +533,8 @@ class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta): deprecated={"squared_loss", "log"}, ) ], - "n_jobs": [None, Integral], - "class_weight": [StrOptions({"balanced"}), dict, None], # a bit tricky? + "n_jobs": [Integral, None], + "class_weight": [StrOptions({"balanced"}), dict, None], } @abstractmethod @@ -857,6 +857,9 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): self : object Returns an instance of self. """ + if not hasattr(self, "classes_"): + self._validate_params() + self._revalidate_params(for_partial_fit=True) if self.class_weight in ["balanced"]: raise ValueError( @@ -912,6 +915,8 @@ def fit(self, X, y, coef_init=None, intercept_init=None, sample_weight=None): self : object Returns an instance of self. """ + self._validate_params() + return self._fit( X, y, @@ -1192,8 +1197,6 @@ class SGDClassifier(BaseSGDClassifier): [1] """ - _parameter_constraints = BaseSGDClassifier._parameter_constraints - def __init__( self, loss="hinge", From 3bd2cc699a51a33d917ca3fd1d460f11625767cb Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 24 Jun 2022 13:40:32 +0200 Subject: [PATCH 30/35] try to leverage inheritence + some fixes --- sklearn/linear_model/_passive_aggressive.py | 71 ++++------ sklearn/linear_model/_perceptron.py | 29 ++-- sklearn/linear_model/_stochastic_gradient.py | 129 +++++++++--------- .../tests/test_passive_aggressive.py | 14 -- sklearn/linear_model/tests/test_sgd.py | 44 +----- sklearn/tests/test_common.py | 2 - sklearn/utils/estimator_checks.py | 1 - 7 files changed, 108 insertions(+), 182 deletions(-) diff --git a/sklearn/linear_model/_passive_aggressive.py b/sklearn/linear_model/_passive_aggressive.py index 6972985c8c300..6562c326a45bb 100644 --- a/sklearn/linear_model/_passive_aggressive.py +++ b/sklearn/linear_model/_passive_aggressive.py @@ -175,21 +175,9 @@ class PassiveAggressiveClassifier(BaseSGDClassifier): """ _parameter_constraints = { + **BaseSGDClassifier._parameter_constraints, "loss": [StrOptions({"hinge", "squared_hinge"})], - "C": [Interval(Real, None, None, closed="neither")], - "fit_intercept": [bool], - "max_iter": [Interval(Integral, 1, None, closed="left")], - "tol": [Interval(Real, None, None, closed="neither"), None], - "shuffle": [bool], - "verbose": [Interval(Integral, 0, None, closed="left")], - "random_state": ["random_state"], - "early_stopping": [bool], - "validation_fraction": [Interval(Real, 0, 1, closed="neither")], - "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], - "warm_start": [bool], - "average": [Interval(Integral, 0, None, closed="left"), bool], - "n_jobs": [None, Integral], - "class_weight": [StrOptions({"balanced"}), dict, None], + "C": [Interval(Real, 0, None, closed="right")], } def __init__( @@ -256,19 +244,23 @@ def partial_fit(self, X, y, classes=None): self : object Fitted estimator. """ - self._revalidate_params(for_partial_fit=True) - if self.class_weight == "balanced": - raise ValueError( - "class_weight 'balanced' is not supported for " - "partial_fit. For 'balanced' weights, use " - "`sklearn.utils.compute_class_weight` with " - "`class_weight='balanced'`. In place of y you " - "can use a large enough subset of the full " - "training set target to properly estimate the " - "class frequency distributions. Pass the " - "resulting weights as the class_weight " - "parameter." - ) + if not hasattr(self, "classes_"): + self._validate_params() + self._more_validate_params(for_partial_fit=True) + + if self.class_weight == "balanced": + raise ValueError( + "class_weight 'balanced' is not supported for " + "partial_fit. For 'balanced' weights, use " + "`sklearn.utils.compute_class_weight` with " + "`class_weight='balanced'`. In place of y you " + "can use a large enough subset of the full " + "training set target to properly estimate the " + "class frequency distributions. Pass the " + "resulting weights as the class_weight " + "parameter." + ) + lr = "pa1" if self.loss == "hinge" else "pa2" return self._partial_fit( X, @@ -306,7 +298,9 @@ def fit(self, X, y, coef_init=None, intercept_init=None): self : object Fitted estimator. """ - self._revalidate_params() + self._validate_params() + self._more_validate_params() + lr = "pa1" if self.loss == "hinge" else "pa2" return self._fit( X, @@ -466,19 +460,9 @@ class PassiveAggressiveRegressor(BaseSGDRegressor): """ _parameter_constraints = { + **BaseSGDRegressor._parameter_constraints, "loss": [StrOptions({"epsilon_insensitive", "squared_epsilon_insensitive"})], - "C": [Interval(Real, None, None, closed="neither")], - "fit_intercept": [bool], - "max_iter": [Interval(Integral, 1, None, closed="left")], - "tol": [Interval(Real, None, None, closed="neither"), None], - "shuffle": [bool], - "verbose": [Interval(Integral, 0, None, closed="left")], - "random_state": ["random_state"], - "early_stopping": [bool], - "validation_fraction": [Interval(Real, 0, 1, closed="neither")], - "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], - "warm_start": [bool], - "average": [Interval(Integral, 0, None, closed="left"), bool], + "C": [Interval(Real, 0, None, closed="right")], "epsilon": [Interval(Real, 0, None, closed="left")], } @@ -536,7 +520,10 @@ def partial_fit(self, X, y): self : object Fitted estimator. """ - self._revalidate_params(for_partial_fit=True) + if not hasattr(self, "coef_"): + self._validate_params() + self._more_validate_params(for_partial_fit=True) + lr = "pa1" if self.loss == "epsilon_insensitive" else "pa2" return self._partial_fit( X, @@ -574,6 +561,8 @@ def fit(self, X, y, coef_init=None, intercept_init=None): Fitted estimator. """ self._validate_params() + self._more_validate_params() + lr = "pa1" if self.loss == "epsilon_insensitive" else "pa2" return self._fit( X, diff --git a/sklearn/linear_model/_perceptron.py b/sklearn/linear_model/_perceptron.py index 6b13d80a12b9c..568b249498531 100644 --- a/sklearn/linear_model/_perceptron.py +++ b/sklearn/linear_model/_perceptron.py @@ -166,24 +166,17 @@ class Perceptron(BaseSGDClassifier): 0.939... """ - _parameter_constraints = { - "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None], - "alpha": [Interval(Real, 0, None, closed="left")], - "l1_ratio": [Interval(Real, 0, 1, closed="both")], - "fit_intercept": [bool], - "max_iter": [Interval(Integral, 1, None, closed="left")], - "tol": [Interval(Real, 0, None, closed="left"), None], - "shuffle": [bool], - "verbose": [Interval(Integral, 0, None, closed="left")], - "eta0": [Interval(Real, 0, None, closed="left")], - "random_state": ["random_state"], - "early_stopping": [bool], - "validation_fraction": [Interval(Real, 0, 1, closed="neither")], - "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], - "warm_start": [bool], - "n_jobs": [None, Integral], - "class_weight": [StrOptions({"balanced"}), dict, None], # a bit tricky? - } + _parameter_constraints = {**BaseSGDClassifier._parameter_constraints} + _parameter_constraints.pop("loss") + _parameter_constraints.pop("average") + _parameter_constraints.update( + { + "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None], + "alpha": [Interval(Real, 0, None, closed="left")], + "l1_ratio": [Interval(Real, 0, 1, closed="both")], + "eta0": [Interval(Real, 0, None, closed="left")], + } + ) def __init__( self, diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index e2dbaab3db687..496c3f3d8cf4e 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -80,27 +80,14 @@ class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta): """Base class for SGD classification and regression.""" _parameter_constraints = { - "penalty": [StrOptions({"l2", "l1", "elasticnet"})], - "alpha": [Interval(Real, 0, None, closed="left")], - # "C": [Interval(Real , 0, None, closed="right")], - "l1_ratio": [Interval(Real, 0, 1, closed="both")], "fit_intercept": ["boolean"], "max_iter": [Interval(Integral, 1, None, closed="left")], - "tol": [Interval(Real, 0, None, closed="left")], + "tol": [Interval(Real, 0, None, closed="left"), None], "shuffle": ["boolean"], "verbose": ["verbose"], - "epsilon": [Interval(Real, 0, None, closed="left")], "random_state": ["random_state"], - "learning_rate": [ - StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}) - ], - "eta0": [Interval(Real, 0, None, closed="left")], - "power_t": [Interval(Real, None, None, closed="neither")], - "early_stopping": ["boolean"], - "validation_fraction": [Interval(Real, 0, 1, closed="neither")], - "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], "warm_start": ["boolean"], - "average": [Interval(Integral, 1, None, closed="left"), bool, np.bool_], + "average": [Interval(Integral, 0, None, closed="left"), bool, np.bool_], } def __init__( @@ -152,7 +139,7 @@ def __init__( def fit(self, X, y): """Fit model.""" - def _revalidate_params(self, for_partial_fit=False): + def _more_validate_params(self, for_partial_fit=False): """Validate input params.""" if self.early_stopping and for_partial_fit: raise ValueError("early_stopping should be False with partial_fit") @@ -517,22 +504,13 @@ class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta): **BaseSGD._parameter_constraints, "loss": [ StrOptions( - { - "hinge", - "squared_hinge", - "perceptron", - "log_loss", - "log", - "modified_huber", - "squared_error", - "squared_loss", - "huber", - "epsilon_insensitive", - "squared_epsilon_insensitive", - }, + set(loss_functions.keys()), deprecated={"squared_loss", "log"}, ) ], + "early_stopping": ["boolean"], + "validation_fraction": [Interval(Real, 0, 1, closed="neither")], + "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], "n_jobs": [Integral, None], "class_weight": [StrOptions({"balanced"}), dict, None], } @@ -680,7 +658,6 @@ def _fit( intercept_init=None, sample_weight=None, ): - self._revalidate_params() if hasattr(self, "classes_"): # delete the attribute otherwise _partial_fit thinks it's not the first call delattr(self, "classes_") @@ -859,20 +836,21 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): """ if not hasattr(self, "classes_"): self._validate_params() + self._more_validate_params(for_partial_fit=True) + + if self.class_weight in ["balanced"]: + raise ValueError( + "class_weight '{0}' is not supported for " + "partial_fit. In order to use 'balanced' weights," + " use compute_class_weight('{0}', " + "classes=classes, y=y). " + "In place of y you can use a large enough sample " + "of the full training set target to properly " + "estimate the class frequency distributions. " + "Pass the resulting weights as the class_weight " + "parameter.".format(self.class_weight) + ) - self._revalidate_params(for_partial_fit=True) - if self.class_weight in ["balanced"]: - raise ValueError( - "class_weight '{0}' is not supported for " - "partial_fit. In order to use 'balanced' weights," - " use compute_class_weight('{0}', " - "classes=classes, y=y). " - "In place of y you can us a large enough sample " - "of the full training set target to properly " - "estimate the class frequency distributions. " - "Pass the resulting weights as the class_weight " - "parameter.".format(self.class_weight) - ) return self._partial_fit( X, y, @@ -916,6 +894,7 @@ def fit(self, X, y, coef_init=None, intercept_init=None, sample_weight=None): Returns an instance of self. """ self._validate_params() + self._more_validate_params() return self._fit( X, @@ -1197,6 +1176,19 @@ class SGDClassifier(BaseSGDClassifier): [1] """ + _parameter_constraints = { + **BaseSGDClassifier._parameter_constraints, + "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None], + "alpha": [Interval(Real, 0, None, closed="left")], + "l1_ratio": [Interval(Real, 0, 1, closed="both")], + "power_t": [Interval(Real, None, None, closed="neither")], + "epsilon": [Interval(Real, 0, None, closed="left")], + "learning_rate": [ + StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}) + ], + "eta0": [Interval(Real, 0, None, closed="left")], + } + def __init__( self, loss="hinge", @@ -1386,16 +1378,13 @@ class BaseSGDRegressor(RegressorMixin, BaseSGD): **BaseSGD._parameter_constraints, "loss": [ StrOptions( - { - "squared_error", - "squared_loss", - "huber", - "epsilon_insensitive", - "squared_epsilon_insensitive", - }, + set(loss_functions.keys()), deprecated={"squared_loss"}, ) ], + "early_stopping": ["boolean"], + "validation_fraction": [Interval(Real, 0, 1, closed="neither")], + "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], } @abstractmethod @@ -1512,7 +1501,10 @@ def partial_fit(self, X, y, sample_weight=None): self : object Returns an instance of self. """ - self._revalidate_params(for_partial_fit=True) + if not hasattr(self, "coef_"): + self._validate_params() + self._more_validate_params(for_partial_fit=True) + return self._partial_fit( X, y, @@ -1538,8 +1530,6 @@ def _fit( intercept_init=None, sample_weight=None, ): - self._revalidate_params() - if self.warm_start and getattr(self, "coef_", None) is not None: if coef_init is None: coef_init = self.coef_ @@ -1604,6 +1594,8 @@ def fit(self, X, y, coef_init=None, intercept_init=None, sample_weight=None): self : object Fitted `SGDRegressor` estimator. """ + self._validate_params() + self._more_validate_params() return self._fit( X, @@ -1958,6 +1950,19 @@ class SGDRegressor(BaseSGDRegressor): ('sgdregressor', SGDRegressor())]) """ + _parameter_constraints = { + **BaseSGDRegressor._parameter_constraints, + "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None], + "alpha": [Interval(Real, 0, None, closed="left")], + "l1_ratio": [Interval(Real, 0, 1, closed="both")], + "power_t": [Interval(Real, None, None, closed="neither")], + "learning_rate": [ + StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}) + ], + "epsilon": [Interval(Real, 0, None, closed="left")], + "eta0": [Interval(Real, 0, None, closed="left")], + } + def __init__( self, loss="squared_error", @@ -2156,21 +2161,13 @@ class SGDOneClassSVM(BaseSGD, OutlierMixin): loss_functions = {"hinge": (Hinge, 1.0)} _parameter_constraints = { - # **BaseSGD._parameter_constraints, + **BaseSGD._parameter_constraints, "nu": [Interval(Real, 0.0, 1.0, closed="right")], - "fit_intercept": [bool], - "max_iter": [Interval(Integral, 1, None, closed="left")], - "tol": [Interval(Real, None, None, closed="left"), None], - "shuffle": [bool], - "verbose": [Interval(Integral, 0, None, closed="left")], - "random_state": ["random_state"], "learning_rate": [ StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}) ], "eta0": [Interval(Real, 0, None, closed="left")], "power_t": [Interval(Real, None, None, closed="neither")], - "warm_start": [bool], - "average": [Interval(Integral, 0, None, closed="left"), bool], } def __init__( @@ -2387,8 +2384,10 @@ def partial_fit(self, X, y=None, sample_weight=None): self : object Returns a fitted instance of self. """ + if not hasattr(self, "coef_"): + self._validate_params() + self._more_validate_params(for_partial_fit=True) - self._revalidate_params(for_partial_fit=True) alpha = self.nu / 2 return self._partial_fit( X, @@ -2483,7 +2482,9 @@ def fit(self, X, y=None, coef_init=None, offset_init=None, sample_weight=None): self : object Returns a fitted instance of self. """ - self._revalidate_params() + self._validate_params() + self._more_validate_params() + alpha = self.nu / 2 self._fit( X, diff --git a/sklearn/linear_model/tests/test_passive_aggressive.py b/sklearn/linear_model/tests/test_passive_aggressive.py index 72c2c31500611..06b6bd5b84cb1 100644 --- a/sklearn/linear_model/tests/test_passive_aggressive.py +++ b/sklearn/linear_model/tests/test_passive_aggressive.py @@ -204,20 +204,6 @@ def test_wrong_class_weight_label(): clf.fit(X2, y2) -def test_wrong_class_weight_format(): - # ValueError due to wrong class_weight argument type. - X2 = np.array([[-1.0, -1.0], [-1.0, 0], [-0.8, -1.0], [1.0, 1.0], [1.0, 0.0]]) - y2 = [1, 1, 1, -1, -1] - - clf = PassiveAggressiveClassifier(class_weight=[0.5], max_iter=100) - with pytest.raises(ValueError): - clf.fit(X2, y2) - - clf = PassiveAggressiveClassifier(class_weight="the larch", max_iter=100) - with pytest.raises(ValueError): - clf.fit(X2, y2) - - def test_regressor_mse(): y_bin = y.copy() y_bin[y != 1] = -1 diff --git a/sklearn/linear_model/tests/test_sgd.py b/sklearn/linear_model/tests/test_sgd.py index a2c956c6cb32b..5c130ce078bb0 100644 --- a/sklearn/linear_model/tests/test_sgd.py +++ b/sklearn/linear_model/tests/test_sgd.py @@ -215,46 +215,6 @@ def asgd(klass, X, y, eta, alpha, weight_init=None, intercept_init=0.0): return average_weights, average_intercept -@pytest.mark.parametrize( - "klass", - [ - SGDClassifier, - SparseSGDClassifier, - SGDRegressor, - SparseSGDRegressor, - SGDOneClassSVM, - SparseSGDOneClassSVM, - ], -) -@pytest.mark.parametrize("fit_method", ["fit", "partial_fit"]) -@pytest.mark.parametrize( - "params, err_msg", - [ - ({"alpha": 0, "learning_rate": "optimal"}, "alpha must be > 0"), - ({"eta0": 0, "learning_rate": "constant"}, "eta0 must be > 0"), - ], - # Avoid long error messages in test names: - # https://github.com/scikit-learn/scikit-learn/issues/21362 - ids=lambda x: x[:10].replace("]", "") if isinstance(x, str) else x, -) -def test_sgd_estimator_params_validation(klass, fit_method, params, err_msg): - """Validate parameters in the different SGD estimators.""" - try: - sgd_estimator = klass(**params) - except TypeError as err: - if "unexpected keyword argument" in str(err): - # skip test if the parameter is not supported by the estimator - return - raise err - - with pytest.raises(ValueError, match=err_msg): - if is_classifier(sgd_estimator) and fit_method == "partial_fit": - fit_params = {"classes": np.unique(Y)} - else: - fit_params = {} - getattr(sgd_estimator, fit_method)(X, Y, **fit_params) - - def _test_warm_start(klass, X, Y, lr): # Test that explicit warm restart... clf = klass(alpha=0.01, eta0=0.01, shuffle=False, learning_rate=lr) @@ -654,7 +614,7 @@ def test_partial_fit_weight_class_balanced(klass): r"class_weight 'balanced' is not supported for " r"partial_fit\. In order to use 'balanced' weights, " r"use compute_class_weight\('balanced', classes=classes, y=y\). " - r"In place of y you can us a large enough sample " + r"In place of y you can use a large enough sample " r"of the full training set target to properly " r"estimate the class frequency distributions\. " r"Pass the resulting weights as the class_weight " @@ -1725,7 +1685,7 @@ def test_ocsvm_vs_sgdocsvm(): fit_intercept=True, max_iter=max_iter, random_state=random_state, - tol=-np.inf, + tol=None, ) pipe_sgd = make_pipeline(transform, clf_sgd) pipe_sgd.fit(X_train) diff --git a/sklearn/tests/test_common.py b/sklearn/tests/test_common.py index 12f9a213f22ea..a6750ff9078a1 100644 --- a/sklearn/tests/test_common.py +++ b/sklearn/tests/test_common.py @@ -572,8 +572,6 @@ def test_estimators_do_not_raise_errors_in_init_or_set_params(Estimator): "PLSCanonical", "PLSRegression", "PLSSVD", - "PassiveAggressiveClassifier", - "PassiveAggressiveRegressor", "PatchExtractor", "PoissonRegressor", "PolynomialCountSketch", diff --git a/sklearn/utils/estimator_checks.py b/sklearn/utils/estimator_checks.py index 30f267c12cdb2..63be54d2db03b 100644 --- a/sklearn/utils/estimator_checks.py +++ b/sklearn/utils/estimator_checks.py @@ -4055,7 +4055,6 @@ def check_param_validation(name, estimator_orig): err_msg = ( f"Mismatch between _parameter_constraints and the parameters of {name}." ) - assert estimator_orig._parameter_constraints.keys() == estimator_params, err_msg # this object does not have a valid type for sure for all params From cc7a30f943643d52050caf37b2d55529dcdeae0e Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 24 Jun 2022 13:49:37 +0200 Subject: [PATCH 31/35] cln --- sklearn/linear_model/_stochastic_gradient.py | 3 +++ sklearn/linear_model/tests/test_sgd.py | 28 +++++++++++--------- sklearn/utils/_param_validation.py | 1 + 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 496c3f3d8cf4e..f0689cacade8d 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -484,6 +484,7 @@ def fit_binary( class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta): + # TODO(1.2): Remove "squared_loss" # TODO(1.3): Remove "log"" loss_functions = { @@ -1365,6 +1366,7 @@ def _more_tags(self): class BaseSGDRegressor(RegressorMixin, BaseSGD): + # TODO: Remove squared_loss in v1.2 loss_functions = { "squared_error": (SquaredLoss,), @@ -2185,6 +2187,7 @@ def __init__( warm_start=False, average=False, ): + alpha = nu / 2 self.nu = nu super(SGDOneClassSVM, self).__init__( diff --git a/sklearn/linear_model/tests/test_sgd.py b/sklearn/linear_model/tests/test_sgd.py index 5c130ce078bb0..b665abf6c31bd 100644 --- a/sklearn/linear_model/tests/test_sgd.py +++ b/sklearn/linear_model/tests/test_sgd.py @@ -1,28 +1,29 @@ import pickle -from unittest.mock import Mock import joblib -import numpy as np import pytest +import numpy as np import scipy.sparse as sp +from unittest.mock import Mock + +from sklearn.utils._testing import assert_allclose +from sklearn.utils._testing import assert_array_equal +from sklearn.utils._testing import assert_almost_equal +from sklearn.utils._testing import assert_array_almost_equal +from sklearn.utils._testing import ignore_warnings from sklearn import linear_model, datasets, metrics from sklearn.base import clone, is_classifier -from sklearn.exceptions import ConvergenceWarning +from sklearn.svm import OneClassSVM +from sklearn.preprocessing import LabelEncoder, scale, MinMaxScaler +from sklearn.preprocessing import StandardScaler from sklearn.kernel_approximation import Nystroem +from sklearn.pipeline import make_pipeline +from sklearn.exceptions import ConvergenceWarning +from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit from sklearn.linear_model import _sgd_fast as sgd_fast from sklearn.linear_model import _stochastic_gradient from sklearn.model_selection import RandomizedSearchCV -from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit -from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import LabelEncoder, scale, MinMaxScaler -from sklearn.preprocessing import StandardScaler -from sklearn.svm import OneClassSVM -from sklearn.utils._testing import assert_allclose -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import ignore_warnings def _update_kwargs(kwargs): @@ -2080,6 +2081,7 @@ def test_SGDClassifier_fit_for_all_backends(backend): ], ) def test_loss_deprecated(old_loss, new_loss, Estimator): + # Note: class BaseSGD calls self._validate_params() in __init__, therefore # even instantiation of class raises FutureWarning for deprecated losses. with pytest.warns(FutureWarning, match=f"The loss '{old_loss}' was deprecated"): diff --git a/sklearn/utils/_param_validation.py b/sklearn/utils/_param_validation.py index 1d0108c93b619..e3118a07b5bfd 100644 --- a/sklearn/utils/_param_validation.py +++ b/sklearn/utils/_param_validation.py @@ -148,6 +148,7 @@ def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): + func_sig = signature(func) # Map *args/**kwargs to the function signature From c5cb17ff046198858a97db5a1bfec49f4a0cf15d Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 24 Jun 2022 13:54:11 +0200 Subject: [PATCH 32/35] lint --- sklearn/linear_model/_passive_aggressive.py | 2 +- sklearn/linear_model/_perceptron.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/linear_model/_passive_aggressive.py b/sklearn/linear_model/_passive_aggressive.py index 6562c326a45bb..0b029fe781376 100644 --- a/sklearn/linear_model/_passive_aggressive.py +++ b/sklearn/linear_model/_passive_aggressive.py @@ -1,6 +1,6 @@ # Authors: Rob Zinkov, Mathieu Blondel # License: BSD 3 clause -from numbers import Integral, Real +from numbers import Real from ._stochastic_gradient import BaseSGDClassifier from ._stochastic_gradient import BaseSGDRegressor diff --git a/sklearn/linear_model/_perceptron.py b/sklearn/linear_model/_perceptron.py index 568b249498531..7d1b401b177de 100644 --- a/sklearn/linear_model/_perceptron.py +++ b/sklearn/linear_model/_perceptron.py @@ -1,6 +1,6 @@ # Author: Mathieu Blondel # License: BSD 3 clause -from numbers import Integral, Real +from numbers import Real from ._stochastic_gradient import BaseSGDClassifier from ..utils._param_validation import StrOptions, Interval From 3e146dc480bd667106e65d3e92896c94e9adb5ab Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 24 Jun 2022 14:12:25 +0200 Subject: [PATCH 33/35] hidden for undocumented learning_rates --- sklearn/linear_model/_perceptron.py | 2 +- sklearn/linear_model/_stochastic_gradient.py | 14 +++++++++----- sklearn/linear_model/tests/test_logistic.py | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/sklearn/linear_model/_perceptron.py b/sklearn/linear_model/_perceptron.py index 7d1b401b177de..d47e17a92d08b 100644 --- a/sklearn/linear_model/_perceptron.py +++ b/sklearn/linear_model/_perceptron.py @@ -39,7 +39,7 @@ class Perceptron(BaseSGDClassifier): .. versionadded:: 0.19 - tol : float, default=1e-3 + tol : float or None, default=1e-3 The stopping criterion. If it is not None, the iterations will stop when (loss > previous_loss - tol). diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index f0689cacade8d..1d505acb6ee5b 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -25,6 +25,7 @@ from ..utils.validation import check_is_fitted, _check_sample_weight from ..utils._param_validation import Interval from ..utils._param_validation import StrOptions +from ..utils._param_validation import Hidden from ..utils.fixes import delayed from ..exceptions import ConvergenceWarning from ..model_selection import StratifiedShuffleSplit, ShuffleSplit @@ -995,7 +996,7 @@ class SGDClassifier(BaseSGDClassifier): .. versionadded:: 0.19 - tol : float, default=1e-3 + tol : float or None, default=1e-3 The stopping criterion. If it is not None, training will stop when (loss > best_loss - tol) for ``n_iter_no_change`` consecutive epochs. @@ -1185,7 +1186,8 @@ class SGDClassifier(BaseSGDClassifier): "power_t": [Interval(Real, None, None, closed="neither")], "epsilon": [Interval(Real, 0, None, closed="left")], "learning_rate": [ - StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}) + StrOptions({"constant", "optimal", "invscaling", "adaptive"}), + Hidden(StrOptions({"pa1", "pa2"})), ], "eta0": [Interval(Real, 0, None, closed="left")], } @@ -1800,7 +1802,7 @@ class SGDRegressor(BaseSGDRegressor): .. versionadded:: 0.19 - tol : float, default=1e-3 + tol : float or None, default=1e-3 The stopping criterion. If it is not None, training will stop when (loss > best_loss - tol) for ``n_iter_no_change`` consecutive epochs. @@ -1959,7 +1961,8 @@ class SGDRegressor(BaseSGDRegressor): "l1_ratio": [Interval(Real, 0, 1, closed="both")], "power_t": [Interval(Real, None, None, closed="neither")], "learning_rate": [ - StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}) + StrOptions({"constant", "optimal", "invscaling", "adaptive"}), + Hidden(StrOptions({"pa1", "pa2"})), ], "epsilon": [Interval(Real, 0, None, closed="left")], "eta0": [Interval(Real, 0, None, closed="left")], @@ -2166,7 +2169,8 @@ class SGDOneClassSVM(BaseSGD, OutlierMixin): **BaseSGD._parameter_constraints, "nu": [Interval(Real, 0.0, 1.0, closed="right")], "learning_rate": [ - StrOptions({"constant", "optimal", "invscaling", "adaptive", "pa1", "pa2"}) + StrOptions({"constant", "optimal", "invscaling", "adaptive"}), + Hidden(StrOptions({"pa1", "pa2"})), ], "eta0": [Interval(Real, 0, None, closed="left")], "power_t": [Interval(Real, None, None, closed="neither")], diff --git a/sklearn/linear_model/tests/test_logistic.py b/sklearn/linear_model/tests/test_logistic.py index 4c8c9daf78731..a4598e709a54f 100644 --- a/sklearn/linear_model/tests/test_logistic.py +++ b/sklearn/linear_model/tests/test_logistic.py @@ -1771,7 +1771,7 @@ def test_elastic_net_versus_sgd(C, l1_ratio): penalty="elasticnet", random_state=1, fit_intercept=False, - tol=-np.inf, + tol=None, max_iter=2000, l1_ratio=l1_ratio, alpha=1.0 / C / n_samples, From 782fad9bd62db782e72ac5cc1194c570f73d83f5 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 24 Jun 2022 15:52:17 +0200 Subject: [PATCH 34/35] address review comments --- sklearn/linear_model/_stochastic_gradient.py | 23 ++++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 1d505acb6ee5b..1f71888b05eb5 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -144,9 +144,11 @@ def _more_validate_params(self, for_partial_fit=False): """Validate input params.""" if self.early_stopping and for_partial_fit: raise ValueError("early_stopping should be False with partial_fit") - if self.learning_rate in ("constant", "invscaling", "adaptive"): - if self.eta0 <= 0.0: - raise ValueError("eta0 must be > 0") + if ( + self.learning_rate in ("constant", "invscaling", "adaptive") + and self.eta0 <= 0.0 + ): + raise ValueError("eta0 must be > 0") if self.learning_rate == "optimal" and self.alpha == 0: raise ValueError( "alpha must be > 0 since " @@ -176,14 +178,11 @@ def _more_validate_params(self, for_partial_fit=False): def _get_loss_function(self, loss): """Get concrete ``LossFunction`` object for str ``loss``.""" - try: - loss_ = self.loss_functions[loss] - loss_class, args = loss_[0], loss_[1:] - if loss in ("huber", "epsilon_insensitive", "squared_epsilon_insensitive"): - args = (self.epsilon,) - return loss_class(*args) - except KeyError as e: - raise ValueError("The loss %s is not supported. " % loss) from e + loss_ = self.loss_functions[loss] + loss_class, args = loss_[0], loss_[1:] + if loss in ("huber", "epsilon_insensitive", "squared_epsilon_insensitive"): + args = (self.epsilon,) + return loss_class(*args) def _get_learning_rate_type(self, learning_rate): return LEARNING_RATE_TYPES[learning_rate] @@ -840,7 +839,7 @@ def partial_fit(self, X, y, classes=None, sample_weight=None): self._validate_params() self._more_validate_params(for_partial_fit=True) - if self.class_weight in ["balanced"]: + if self.class_weight == "balanced": raise ValueError( "class_weight '{0}' is not supported for " "partial_fit. In order to use 'balanced' weights," From c14410da81263d61560da88fa1969a29a732a596 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Fri, 24 Jun 2022 16:57:27 +0200 Subject: [PATCH 35/35] Apply suggestions from code review Co-authored-by: Meekail Zain <34613774+Micky774@users.noreply.github.com> --- sklearn/linear_model/_stochastic_gradient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py index 1f71888b05eb5..ad69a0b828f6b 100644 --- a/sklearn/linear_model/_stochastic_gradient.py +++ b/sklearn/linear_model/_stochastic_gradient.py @@ -505,7 +505,7 @@ class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta): **BaseSGD._parameter_constraints, "loss": [ StrOptions( - set(loss_functions.keys()), + set(loss_functions), deprecated={"squared_loss", "log"}, ) ], @@ -1381,7 +1381,7 @@ class BaseSGDRegressor(RegressorMixin, BaseSGD): **BaseSGD._parameter_constraints, "loss": [ StrOptions( - set(loss_functions.keys()), + set(loss_functions), deprecated={"squared_loss"}, ) ],