Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions sklearn/linear_model/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ def _preprocess_data(
sample_weight=None,
check_input=True,
rescale_with_sw=True,
center_X=True,
):
"""Common data preprocessing for fitting linear models.

Expand All @@ -132,9 +133,9 @@ def _preprocess_data(

Then, if `fit_intercept=True` this preprocessing centers both `X` and `y` as
follows:
- if `X` is dense, center the data and
- if `X` is dense and `center_X=True`, center the data and
store the mean vector in `X_offset`.
- if `X` is sparse, store the mean in `X_offset`
- if `X` is sparse, or `center_X=False`, store the mean in `X_offset`
without centering `X`. The centering is expected to be handled by the
linear solver where appropriate.
- in either case, always center `y` and store the mean in `y_offset`.
Expand All @@ -147,12 +148,16 @@ def _preprocess_data(
If `rescale_with_sw` is True, then X and y are rescaled with the square root of
sample weights.

`center_X=False` lets a caller with a dense `X` skip the `O(n_samples *
n_features)` centering pass. It has no effect on sparse `X`, which is never
centered in place regardless.

Returns
-------
X_out : {ndarray, sparse matrix} of shape (n_samples, n_features)
If copy=True a copy of the input X is triggered, otherwise operations are
inplace.
If input X is dense, then X_out is centered.
If input X is dense and `center_X=True`, then X_out is centered.
y_out : {ndarray, sparse matrix} of shape (n_samples,) or (n_samples, n_targets)
Centered copy of y.
X_offset : ndarray of shape (n_features,)
Expand Down Expand Up @@ -191,9 +196,10 @@ def _preprocess_data(
X_offset, X_var = mean_variance_axis(X, axis=0, weights=sample_weight)
else:
X_offset = _average(X, axis=0, weights=sample_weight, xp=xp)

X_offset = xp.astype(X_offset, X.dtype, copy=False)
X -= X_offset

if center_X:
X -= X_offset

y_offset = _average(y, axis=0, weights=sample_weight, xp=xp)
y -= y_offset
Expand Down
74 changes: 65 additions & 9 deletions sklearn/linear_model/_ridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,14 +212,24 @@ def _solve_lsqr(
return coefs, n_iter


def _solve_cholesky(X, y, alpha):
def _solve_cholesky(X, y, alpha, X_offset=None):
# w = inv(X^t X + alpha*Id) * X.T y
#
# If X_offset is given, X is assumed to *not* be centered (unlike y, which
# is always assumed centered) and the Gram matrix is corrected
# algebraically instead of materializing a centered copy of X:
# Xc.T @ Xc = X.T @ X - n_samples * outer(X_offset, X_offset)
# Xy needs no such correction: Xc.T @ yc = X.T @ yc - X_offset * yc.sum(0),
# and yc.sum(0) is 0 because yc is centered.
n_features = X.shape[1]
n_targets = y.shape[1]

A = safe_sparse_dot(X.T, X, dense_output=True)
Xy = safe_sparse_dot(X.T, y, dense_output=True)

if X_offset is not None:
A -= X.shape[0] * np.outer(X_offset, X_offset)

one_alpha = np.array_equal(alpha, len(alpha) * [alpha[0]])

if one_alpha:
Expand Down Expand Up @@ -379,9 +389,14 @@ def func(w):
return coefs


def _get_valid_accept_sparse(is_X_sparse, solver):
def _get_valid_accept_sparse(is_X_sparse, solver, fit_intercept=False):
if is_X_sparse and solver in ["auto", "sag", "saga"]:
# sag/saga's Cython solver needs actual CSR structure to run.
return "csr"
elif is_X_sparse and fit_intercept:
# when `fit_intercept=True`, `mean_variance_axis` will be called on X
# and it requires csr/csc format
return ["csr", "csc"]
Comment on lines +396 to +399

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note to reviewers: this addition is needed because I now pass check_input=False to _preprocess_data which would have done this logic.

else:
return ["csr", "csc", "coo"]

Expand Down Expand Up @@ -755,8 +770,12 @@ def _ridge_regression(
solver = "svd"
else:
try:
coef = _solve_cholesky(X, y, alpha)
coef = _solve_cholesky(X, y, alpha, X_offset=X_offset)
except linalg.LinAlgError:
if X_offset is not None:
# X was left uncentered to avoid materializing a copy on
# the svd fallback needs centered X:
X = X - X_offset
# use SVD solver if matrix is singular
solver = "svd"

Expand Down Expand Up @@ -960,17 +979,41 @@ def fit(self, X, y, sample_weight=None):
if sample_weight is not None:
sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)

# when X is sparse we only remove offset from y
X_is_sparse = sparse.issparse(X)
solver = resolve_solver(
solver, self.positive, return_intercept=False, is_sparse=X_is_sparse, xp=xp
)

# X with n_features <= n_samples, no sample weights, and a
# resolved "cholesky" solver hits _solve_cholesky's primal branch,
# which can apply an algebraically-centering optimization.
use_no_center_cholesky = (
self.fit_intercept
and sample_weight is None
and solver == "cholesky"
and X.shape[0] >= X.shape[1]
)

X, y, X_offset, y_offset, X_scale, _ = _preprocess_data(
X,
y,
fit_intercept=self.fit_intercept,
copy=self.copy_X,
copy=(
self.copy_X
# If `use_no_center_cholesky` or X is sparse,
# X is never mutated
and not use_no_center_cholesky
and not X_is_sparse
),
# X and y were already validated by `validate_data` in the
# public `fit` method of the subclass (e.g. `Ridge.fit`).
check_input=False,
Comment thread
cakedev0 marked this conversation as resolved.
sample_weight=sample_weight,
rescale_with_sw=False,
center_X=not use_no_center_cholesky,
)

if solver == "sag" and sparse.issparse(X) and self.fit_intercept:
if solver == "sag" and X_is_sparse and self.fit_intercept:
self.coef_, self.n_iter_, self.intercept_, self.solver_ = _ridge_regression(
X,
y,
Expand All @@ -990,9 +1033,13 @@ def fit(self, X, y, sample_weight=None):
self.intercept_ += y_offset

else:
if sparse.issparse(X) and self.fit_intercept:
if X_is_sparse and self.fit_intercept:
# required to fit intercept with sparse_cg and lbfgs solver
params = {"X_offset": X_offset, "X_scale": X_scale}
elif use_no_center_cholesky:
# X was left uncentered; _solve_cholesky applies the
# correction algebraically from X_offset.
params = {"X_offset": X_offset}
else:
# for dense matrices or when intercept is set to 0
params = {}
Expand Down Expand Up @@ -1246,7 +1293,9 @@ def fit(self, X, y, sample_weight=None):
self : object
Fitted estimator.
"""
_accept_sparse = _get_valid_accept_sparse(sparse.issparse(X), self.solver)
_accept_sparse = _get_valid_accept_sparse(
sparse.issparse(X), self.solver, self.fit_intercept
)
xp, _, device = get_namespace_and_device(X)
y, sample_weight = move_to(y, sample_weight, xp=xp, device=device)

Expand Down Expand Up @@ -1320,14 +1369,21 @@ def _prepare_data(self, X, y, sample_weight, solver):
Y : ndarray of shape (n_samples, n_classes)
The binarized version of `y`.
"""
accept_sparse = _get_valid_accept_sparse(sparse.issparse(X), solver)
accept_sparse = _get_valid_accept_sparse(
sparse.issparse(X), solver, self.fit_intercept
)
xp, _, device = get_namespace_and_device(X)
sample_weight = move_to(sample_weight, xp=xp, device=device)
X, y = validate_data(
self,
X,
y,
accept_sparse=accept_sparse,
# X (not y: dtype here only ever applies to X, never to the
# class labels) needs to already be a supported float dtype by
# the time it reaches `_BaseRidge.fit`, which trusts this
# validation and skips its own for dense X.
dtype=[xp.float64, xp.float32],
multi_output=True,
y_numeric=False,
force_writeable=True,
Expand Down
Loading