diff --git a/sklearn/linear_model/_base.py b/sklearn/linear_model/_base.py index 3acb42f35d894..22ae8b8bde8f8 100644 --- a/sklearn/linear_model/_base.py +++ b/sklearn/linear_model/_base.py @@ -119,6 +119,7 @@ def _preprocess_data( sample_weight=None, check_input=True, rescale_with_sw=True, + center_X=True, ): """Common data preprocessing for fitting linear models. @@ -132,9 +133,9 @@ def _preprocess_data( Then, if `fit_intercept=True` this preprocessing centers both `X` and `y` as follows: - - if `X` is dense, center the data and + - if `X` is dense and `center_X=True`, center the data and store the mean vector in `X_offset`. - - if `X` is sparse, store the mean in `X_offset` + - if `X` is sparse, or `center_X=False`, store the mean in `X_offset` without centering `X`. The centering is expected to be handled by the linear solver where appropriate. - in either case, always center `y` and store the mean in `y_offset`. @@ -147,12 +148,16 @@ def _preprocess_data( If `rescale_with_sw` is True, then X and y are rescaled with the square root of sample weights. + `center_X=False` lets a caller with a dense `X` skip the `O(n_samples * + n_features)` centering pass. It has no effect on sparse `X`, which is never + centered in place regardless. + Returns ------- X_out : {ndarray, sparse matrix} of shape (n_samples, n_features) If copy=True a copy of the input X is triggered, otherwise operations are inplace. - If input X is dense, then X_out is centered. + If input X is dense and `center_X=True`, then X_out is centered. y_out : {ndarray, sparse matrix} of shape (n_samples,) or (n_samples, n_targets) Centered copy of y. X_offset : ndarray of shape (n_features,) @@ -191,9 +196,10 @@ def _preprocess_data( X_offset, X_var = mean_variance_axis(X, axis=0, weights=sample_weight) else: X_offset = _average(X, axis=0, weights=sample_weight, xp=xp) - X_offset = xp.astype(X_offset, X.dtype, copy=False) - X -= X_offset + + if center_X: + X -= X_offset y_offset = _average(y, axis=0, weights=sample_weight, xp=xp) y -= y_offset diff --git a/sklearn/linear_model/_ridge.py b/sklearn/linear_model/_ridge.py index 348e4d2a296c9..3b06b17d73a22 100644 --- a/sklearn/linear_model/_ridge.py +++ b/sklearn/linear_model/_ridge.py @@ -212,14 +212,24 @@ def _solve_lsqr( return coefs, n_iter -def _solve_cholesky(X, y, alpha): +def _solve_cholesky(X, y, alpha, X_offset=None): # w = inv(X^t X + alpha*Id) * X.T y + # + # If X_offset is given, X is assumed to *not* be centered (unlike y, which + # is always assumed centered) and the Gram matrix is corrected + # algebraically instead of materializing a centered copy of X: + # Xc.T @ Xc = X.T @ X - n_samples * outer(X_offset, X_offset) + # Xy needs no such correction: Xc.T @ yc = X.T @ yc - X_offset * yc.sum(0), + # and yc.sum(0) is 0 because yc is centered. n_features = X.shape[1] n_targets = y.shape[1] A = safe_sparse_dot(X.T, X, dense_output=True) Xy = safe_sparse_dot(X.T, y, dense_output=True) + if X_offset is not None: + A -= X.shape[0] * np.outer(X_offset, X_offset) + one_alpha = np.array_equal(alpha, len(alpha) * [alpha[0]]) if one_alpha: @@ -379,9 +389,14 @@ def func(w): return coefs -def _get_valid_accept_sparse(is_X_sparse, solver): +def _get_valid_accept_sparse(is_X_sparse, solver, fit_intercept=False): if is_X_sparse and solver in ["auto", "sag", "saga"]: + # sag/saga's Cython solver needs actual CSR structure to run. return "csr" + elif is_X_sparse and fit_intercept: + # when `fit_intercept=True`, `mean_variance_axis` will be called on X + # and it requires csr/csc format + return ["csr", "csc"] else: return ["csr", "csc", "coo"] @@ -755,8 +770,12 @@ def _ridge_regression( solver = "svd" else: try: - coef = _solve_cholesky(X, y, alpha) + coef = _solve_cholesky(X, y, alpha, X_offset=X_offset) except linalg.LinAlgError: + if X_offset is not None: + # X was left uncentered to avoid materializing a copy on + # the svd fallback needs centered X: + X = X - X_offset # use SVD solver if matrix is singular solver = "svd" @@ -960,17 +979,41 @@ def fit(self, X, y, sample_weight=None): if sample_weight is not None: sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) - # when X is sparse we only remove offset from y + X_is_sparse = sparse.issparse(X) + solver = resolve_solver( + solver, self.positive, return_intercept=False, is_sparse=X_is_sparse, xp=xp + ) + + # X with n_features <= n_samples, no sample weights, and a + # resolved "cholesky" solver hits _solve_cholesky's primal branch, + # which can apply an algebraically-centering optimization. + use_no_center_cholesky = ( + self.fit_intercept + and sample_weight is None + and solver == "cholesky" + and X.shape[0] >= X.shape[1] + ) + X, y, X_offset, y_offset, X_scale, _ = _preprocess_data( X, y, fit_intercept=self.fit_intercept, - copy=self.copy_X, + copy=( + self.copy_X + # If `use_no_center_cholesky` or X is sparse, + # X is never mutated + and not use_no_center_cholesky + and not X_is_sparse + ), + # X and y were already validated by `validate_data` in the + # public `fit` method of the subclass (e.g. `Ridge.fit`). + check_input=False, sample_weight=sample_weight, rescale_with_sw=False, + center_X=not use_no_center_cholesky, ) - if solver == "sag" and sparse.issparse(X) and self.fit_intercept: + if solver == "sag" and X_is_sparse and self.fit_intercept: self.coef_, self.n_iter_, self.intercept_, self.solver_ = _ridge_regression( X, y, @@ -990,9 +1033,13 @@ def fit(self, X, y, sample_weight=None): self.intercept_ += y_offset else: - if sparse.issparse(X) and self.fit_intercept: + if X_is_sparse and self.fit_intercept: # required to fit intercept with sparse_cg and lbfgs solver params = {"X_offset": X_offset, "X_scale": X_scale} + elif use_no_center_cholesky: + # X was left uncentered; _solve_cholesky applies the + # correction algebraically from X_offset. + params = {"X_offset": X_offset} else: # for dense matrices or when intercept is set to 0 params = {} @@ -1246,7 +1293,9 @@ def fit(self, X, y, sample_weight=None): self : object Fitted estimator. """ - _accept_sparse = _get_valid_accept_sparse(sparse.issparse(X), self.solver) + _accept_sparse = _get_valid_accept_sparse( + sparse.issparse(X), self.solver, self.fit_intercept + ) xp, _, device = get_namespace_and_device(X) y, sample_weight = move_to(y, sample_weight, xp=xp, device=device) @@ -1320,7 +1369,9 @@ def _prepare_data(self, X, y, sample_weight, solver): Y : ndarray of shape (n_samples, n_classes) The binarized version of `y`. """ - accept_sparse = _get_valid_accept_sparse(sparse.issparse(X), solver) + accept_sparse = _get_valid_accept_sparse( + sparse.issparse(X), solver, self.fit_intercept + ) xp, _, device = get_namespace_and_device(X) sample_weight = move_to(sample_weight, xp=xp, device=device) X, y = validate_data( @@ -1328,6 +1379,11 @@ def _prepare_data(self, X, y, sample_weight, solver): X, y, accept_sparse=accept_sparse, + # X (not y: dtype here only ever applies to X, never to the + # class labels) needs to already be a supported float dtype by + # the time it reaches `_BaseRidge.fit`, which trusts this + # validation and skips its own for dense X. + dtype=[xp.float64, xp.float32], multi_output=True, y_numeric=False, force_writeable=True,