From a9284790f7636b2c9a292af2ff84bb8fc99991d7 Mon Sep 17 00:00:00 2001 From: Joerg Date: Thu, 16 Nov 2017 13:04:12 +0100 Subject: [PATCH 01/10] quick fix --- sklearn/linear_model/bayes.py | 5 ++++- sklearn/linear_model/tests/test_bayes.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/sklearn/linear_model/bayes.py b/sklearn/linear_model/bayes.py index e754613cda381..96b89f7a331d5 100644 --- a/sklearn/linear_model/bayes.py +++ b/sklearn/linear_model/bayes.py @@ -485,6 +485,7 @@ def fit(self, X, y): sigma_, np.dot(X[:, keep_lambda].T, y)) # Update alpha and lambda + lambda_old = np.copy(lambda_) rmse_ = np.sum((y - np.dot(X, coef_)) ** 2) gamma_ = 1. - lambda_[keep_lambda] * np.diag(sigma_) lambda_[keep_lambda] = ((gamma_ + 2. * lambda_1) / @@ -494,6 +495,7 @@ def fit(self, X, y): (rmse_ + 2. * alpha_2)) # Prune the weights with a precision over a threshold + keep_lambda_old = np.copy(keep_lambda) keep_lambda = lambda_ < self.threshold_lambda coef_[~keep_lambda] = 0 @@ -517,6 +519,7 @@ def fit(self, X, y): self.alpha_ = alpha_ self.sigma_ = sigma_ self.lambda_ = lambda_ + self._keep_lambda_old = keep_lambda_old self._set_intercept(X_offset_, y_offset_, X_scale_) return self @@ -548,7 +551,7 @@ def predict(self, X, return_std=False): else: if self.normalize: X = (X - self.X_offset_) / self.X_scale_ - X = X[:, self.lambda_ < self.threshold_lambda] + X = X[:, self._keep_lambda_old] sigmas_squared_data = (np.dot(X, self.sigma_) * X).sum(axis=1) y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_)) return y_mean, y_std diff --git a/sklearn/linear_model/tests/test_bayes.py b/sklearn/linear_model/tests/test_bayes.py index 5337c0a19c5cf..946d97436c443 100644 --- a/sklearn/linear_model/tests/test_bayes.py +++ b/sklearn/linear_model/tests/test_bayes.py @@ -11,6 +11,7 @@ from sklearn.utils.testing import assert_array_less from sklearn.utils.testing import SkipTest from sklearn.utils import check_random_state +from sklearn.random_projection import sparse_random_matrix from sklearn.linear_model.bayes import BayesianRidge, ARDRegression from sklearn.linear_model import Ridge from sklearn import datasets @@ -110,6 +111,24 @@ def test_std_bayesian_ridge_ard_with_constant_input(): assert_array_less(y_std, expected_upper_boundary) + +def test_regression_issue_10128(): + # this test throws a `ValueError` on master, commit 5963fd2 + from sklearn.random_projection import sparse_random_matrix + np.random.seed(752) + n = 100 + d = 10 + n_samples = 96 + n_features = 9 + X = sparse_random_matrix(n, d, density=0.10).toarray() + X, y = X[: ,:n_features], X[:, -1] + X_train, y_train = X[:n_samples], y[:n_samples] + X_test = np.zeros((1, n_features)) + clf = ARDRegression() + clf.fit(X_train, y_train) + clf.predict(X_test, return_std=True) + + def test_toy_ard_object(): # Test BayesianRegression ARD classifier X = np.array([[1], [2], [3]]) From 1035e98f5bf471d12ea711dd72b21de3e13300b8 Mon Sep 17 00:00:00 2001 From: Joerg Date: Thu, 16 Nov 2017 13:09:22 +0100 Subject: [PATCH 02/10] remove unused helper variable --- sklearn/linear_model/bayes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sklearn/linear_model/bayes.py b/sklearn/linear_model/bayes.py index 96b89f7a331d5..927b407d96f74 100644 --- a/sklearn/linear_model/bayes.py +++ b/sklearn/linear_model/bayes.py @@ -485,7 +485,6 @@ def fit(self, X, y): sigma_, np.dot(X[:, keep_lambda].T, y)) # Update alpha and lambda - lambda_old = np.copy(lambda_) rmse_ = np.sum((y - np.dot(X, coef_)) ** 2) gamma_ = 1. - lambda_[keep_lambda] * np.diag(sigma_) lambda_[keep_lambda] = ((gamma_ + 2. * lambda_1) / From 5330801f987a20c82cc33d7c288789c748d04e05 Mon Sep 17 00:00:00 2001 From: Joerg Date: Thu, 16 Nov 2017 13:13:32 +0100 Subject: [PATCH 03/10] remove double import in test --- sklearn/linear_model/tests/test_bayes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sklearn/linear_model/tests/test_bayes.py b/sklearn/linear_model/tests/test_bayes.py index 946d97436c443..41db42ddfb1b6 100644 --- a/sklearn/linear_model/tests/test_bayes.py +++ b/sklearn/linear_model/tests/test_bayes.py @@ -114,7 +114,6 @@ def test_std_bayesian_ridge_ard_with_constant_input(): def test_regression_issue_10128(): # this test throws a `ValueError` on master, commit 5963fd2 - from sklearn.random_projection import sparse_random_matrix np.random.seed(752) n = 100 d = 10 From 5c6625d6912f8c8159b2260c377fff94ed1e49df Mon Sep 17 00:00:00 2001 From: Joerg Date: Thu, 16 Nov 2017 15:59:01 +0100 Subject: [PATCH 04/10] update docstring, fix blanks --- sklearn/linear_model/tests/test_bayes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sklearn/linear_model/tests/test_bayes.py b/sklearn/linear_model/tests/test_bayes.py index 41db42ddfb1b6..1d3523656289e 100644 --- a/sklearn/linear_model/tests/test_bayes.py +++ b/sklearn/linear_model/tests/test_bayes.py @@ -111,16 +111,15 @@ def test_std_bayesian_ridge_ard_with_constant_input(): assert_array_less(y_std, expected_upper_boundary) - def test_regression_issue_10128(): - # this test throws a `ValueError` on master, commit 5963fd2 + # this ARDRegression test throws a `ValueError` on master, commit 5963fd2 np.random.seed(752) n = 100 d = 10 n_samples = 96 n_features = 9 X = sparse_random_matrix(n, d, density=0.10).toarray() - X, y = X[: ,:n_features], X[:, -1] + X, y = X[:, :n_features], X[:, -1] X_train, y_train = X[:n_samples], y[:n_samples] X_test = np.zeros((1, n_features)) clf = ARDRegression() From 8ec80243bf4597f4cc93e6823adc92221bbb4c59 Mon Sep 17 00:00:00 2001 From: Joerg Date: Fri, 17 Nov 2017 15:56:19 +0100 Subject: [PATCH 05/10] apply update for mu and sigma after iteration loop --- sklearn/linear_model/bayes.py | 18 +++++++++++++++--- sklearn/linear_model/tests/test_bayes.py | 17 +++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/sklearn/linear_model/bayes.py b/sklearn/linear_model/bayes.py index e754613cda381..9fda4f88cfc27 100644 --- a/sklearn/linear_model/bayes.py +++ b/sklearn/linear_model/bayes.py @@ -469,9 +469,8 @@ def fit(self, X, y): self.scores_ = list() coef_old_ = None - # Iterative procedure of ARDRegression - for iter_ in range(self.n_iter): - # Compute mu and sigma (using Woodbury matrix identity) + # Compute sigma and mu (using Woodbury matrix identity) + def update_sigma(X, alpha_, lambda_, keep_lambda, n_samples): sigma_ = pinvh(np.eye(n_samples) / alpha_ + np.dot(X[:, keep_lambda] * np.reshape(1. / lambda_[keep_lambda], [1, -1]), @@ -481,8 +480,17 @@ def fit(self, X, y): sigma_ = - np.dot(np.reshape(1. / lambda_[keep_lambda], [-1, 1]) * X[:, keep_lambda].T, sigma_) sigma_.flat[::(sigma_.shape[1] + 1)] += 1. / lambda_[keep_lambda] + return sigma_ + + def update_coeff(X, y, coef_, alpha_, keep_lambda, sigma_): coef_[keep_lambda] = alpha_ * np.dot( sigma_, np.dot(X[:, keep_lambda].T, y)) + return coef_ + + # Iterative procedure of ARDRegression + for iter_ in range(self.n_iter): + sigma_ = update_sigma(X, alpha_, lambda_, keep_lambda, n_samples) + coef_ = update_coeff(X, y, coef_, alpha_, keep_lambda, sigma_) # Update alpha and lambda rmse_ = np.sum((y - np.dot(X, coef_)) ** 2) @@ -513,6 +521,10 @@ def fit(self, X, y): break coef_old_ = np.copy(coef_) + # update sigma and mu using updated parameters from the last iteration + sigma_ = update_sigma(X, alpha_, lambda_, keep_lambda, n_samples) + coef_ = update_coeff(X, y, coef_, alpha_, keep_lambda, sigma_) + self.coef_ = coef_ self.alpha_ = alpha_ self.sigma_ = sigma_ diff --git a/sklearn/linear_model/tests/test_bayes.py b/sklearn/linear_model/tests/test_bayes.py index 5337c0a19c5cf..1d3523656289e 100644 --- a/sklearn/linear_model/tests/test_bayes.py +++ b/sklearn/linear_model/tests/test_bayes.py @@ -11,6 +11,7 @@ from sklearn.utils.testing import assert_array_less from sklearn.utils.testing import SkipTest from sklearn.utils import check_random_state +from sklearn.random_projection import sparse_random_matrix from sklearn.linear_model.bayes import BayesianRidge, ARDRegression from sklearn.linear_model import Ridge from sklearn import datasets @@ -110,6 +111,22 @@ def test_std_bayesian_ridge_ard_with_constant_input(): assert_array_less(y_std, expected_upper_boundary) +def test_regression_issue_10128(): + # this ARDRegression test throws a `ValueError` on master, commit 5963fd2 + np.random.seed(752) + n = 100 + d = 10 + n_samples = 96 + n_features = 9 + X = sparse_random_matrix(n, d, density=0.10).toarray() + X, y = X[:, :n_features], X[:, -1] + X_train, y_train = X[:n_samples], y[:n_samples] + X_test = np.zeros((1, n_features)) + clf = ARDRegression() + clf.fit(X_train, y_train) + clf.predict(X_test, return_std=True) + + def test_toy_ard_object(): # Test BayesianRegression ARD classifier X = np.array([[1], [2], [3]]) From c2821b62f833b8987be44b14f5f70ec010c94b60 Mon Sep 17 00:00:00 2001 From: Joerg Date: Fri, 17 Nov 2017 16:00:14 +0100 Subject: [PATCH 06/10] restore changes from previous fix --- sklearn/linear_model/bayes.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sklearn/linear_model/bayes.py b/sklearn/linear_model/bayes.py index a1d38d6db0f51..9fda4f88cfc27 100644 --- a/sklearn/linear_model/bayes.py +++ b/sklearn/linear_model/bayes.py @@ -502,7 +502,6 @@ def update_coeff(X, y, coef_, alpha_, keep_lambda, sigma_): (rmse_ + 2. * alpha_2)) # Prune the weights with a precision over a threshold - keep_lambda_old = np.copy(keep_lambda) keep_lambda = lambda_ < self.threshold_lambda coef_[~keep_lambda] = 0 @@ -530,7 +529,6 @@ def update_coeff(X, y, coef_, alpha_, keep_lambda, sigma_): self.alpha_ = alpha_ self.sigma_ = sigma_ self.lambda_ = lambda_ - self._keep_lambda_old = keep_lambda_old self._set_intercept(X_offset_, y_offset_, X_scale_) return self @@ -562,7 +560,7 @@ def predict(self, X, return_std=False): else: if self.normalize: X = (X - self.X_offset_) / self.X_scale_ - X = X[:, self._keep_lambda_old] + X = X[:, self.lambda_ < self.threshold_lambda] sigmas_squared_data = (np.dot(X, self.sigma_) * X).sum(axis=1) y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_)) return y_mean, y_std From 9e9071b59191db556e5053034016ef5f8ec27460 Mon Sep 17 00:00:00 2001 From: Joerg Date: Fri, 17 Nov 2017 17:33:46 +0100 Subject: [PATCH 07/10] update test --- sklearn/linear_model/tests/test_bayes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sklearn/linear_model/tests/test_bayes.py b/sklearn/linear_model/tests/test_bayes.py index 1d3523656289e..32771d541a9ca 100644 --- a/sklearn/linear_model/tests/test_bayes.py +++ b/sklearn/linear_model/tests/test_bayes.py @@ -112,7 +112,7 @@ def test_std_bayesian_ridge_ard_with_constant_input(): def test_regression_issue_10128(): - # this ARDRegression test throws a `ValueError` on master, commit 5963fd2 + # Reproduces the bug in ARDRegression reported in issue #10128 np.random.seed(752) n = 100 d = 10 @@ -124,6 +124,9 @@ def test_regression_issue_10128(): X_test = np.zeros((1, n_features)) clf = ARDRegression() clf.fit(X_train, y_train) + assert_array_almost_equal(clf.sigma_, np.empty(shape=(0, 0))) + assert_array_almost_equal(clf.coef_, np.zeros(shape=n_features)) + # Check that no `ValueError` is thrown clf.predict(X_test, return_std=True) From 7741afc527977559e3673e898828cd598177f06f Mon Sep 17 00:00:00 2001 From: Joerg Date: Fri, 17 Nov 2017 18:13:00 +0100 Subject: [PATCH 08/10] update comment in test --- sklearn/linear_model/tests/test_bayes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sklearn/linear_model/tests/test_bayes.py b/sklearn/linear_model/tests/test_bayes.py index 32771d541a9ca..5fabebf38019c 100644 --- a/sklearn/linear_model/tests/test_bayes.py +++ b/sklearn/linear_model/tests/test_bayes.py @@ -126,7 +126,7 @@ def test_regression_issue_10128(): clf.fit(X_train, y_train) assert_array_almost_equal(clf.sigma_, np.empty(shape=(0, 0))) assert_array_almost_equal(clf.coef_, np.zeros(shape=n_features)) - # Check that no `ValueError` is thrown + # Ensure that no error is thrown clf.predict(X_test, return_std=True) From 5d62e97c16dd6e0ac4d5f283ca26d08aaf34112d Mon Sep 17 00:00:00 2001 From: Joerg Date: Sat, 18 Nov 2017 13:48:35 +0100 Subject: [PATCH 09/10] improve test --- sklearn/linear_model/tests/test_bayes.py | 32 +++++++++++------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/sklearn/linear_model/tests/test_bayes.py b/sklearn/linear_model/tests/test_bayes.py index 5fabebf38019c..443f856fa7285 100644 --- a/sklearn/linear_model/tests/test_bayes.py +++ b/sklearn/linear_model/tests/test_bayes.py @@ -9,9 +9,9 @@ from sklearn.utils.testing import assert_array_almost_equal from sklearn.utils.testing import assert_almost_equal from sklearn.utils.testing import assert_array_less +from sklearn.utils.testing import assert_equal from sklearn.utils.testing import SkipTest from sklearn.utils import check_random_state -from sklearn.random_projection import sparse_random_matrix from sklearn.linear_model.bayes import BayesianRidge, ARDRegression from sklearn.linear_model import Ridge from sklearn import datasets @@ -111,23 +111,19 @@ def test_std_bayesian_ridge_ard_with_constant_input(): assert_array_less(y_std, expected_upper_boundary) -def test_regression_issue_10128(): - # Reproduces the bug in ARDRegression reported in issue #10128 - np.random.seed(752) - n = 100 - d = 10 - n_samples = 96 - n_features = 9 - X = sparse_random_matrix(n, d, density=0.10).toarray() - X, y = X[:, :n_features], X[:, -1] - X_train, y_train = X[:n_samples], y[:n_samples] - X_test = np.zeros((1, n_features)) - clf = ARDRegression() - clf.fit(X_train, y_train) - assert_array_almost_equal(clf.sigma_, np.empty(shape=(0, 0))) - assert_array_almost_equal(clf.coef_, np.zeros(shape=n_features)) - # Ensure that no error is thrown - clf.predict(X_test, return_std=True) +def test_update_of_sigma_in_ard(): + # Checks that `sigma_` is updated correctly after the last iteration + # of the ARDRegression algorithm. See issue #10128. + X = np.array([[1, 0], + [0, 0]]) + y = np.array([0, 0]) + clf = ARDRegression(n_iter=1) + clf.fit(X, y) + # With the inputs above, ARDRegression prunes one of the two coefficients + # in the first iteration. Hence, the expected shape of `sigma_` is (1, 1). + assert_equal(clf.sigma_.shape, (1, 1)) + # Ensure that no error is thrown at prediction stage + clf.predict(X, return_std=True) def test_toy_ard_object(): From 24c03739461030ee69bf35391602efe575673832 Mon Sep 17 00:00:00 2001 From: Joerg Date: Tue, 21 Nov 2017 15:00:51 +0100 Subject: [PATCH 10/10] update whatsnew --- doc/whats_new/v0.20.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/whats_new/v0.20.rst b/doc/whats_new/v0.20.rst index 58506cf8aa99b..4e1d602e68aca 100644 --- a/doc/whats_new/v0.20.rst +++ b/doc/whats_new/v0.20.rst @@ -22,6 +22,7 @@ random sampling procedures. - :class:`neural_network.BaseMultilayerPerceptron` (bug fix) - :class:`neural_network.MLPRegressor` (bug fix) - :class:`neural_network.MLPClassifier` (bug fix) +- :class:`linear_model.ARDRegression` (bug fix) Details are listed in the changelog below. @@ -125,6 +126,10 @@ Classifiers and regressors error for prior list which summed to 1. :issue:`10005` by :user:`Gaurav Dhingra `. +- Fixed a bug in :class:`linear_model.ARDRegression` which caused incorrectly + updated estimates for the standard deviation and the coefficients. + :issue:`10153` by :user:`Jörg Döpfert `. + Decomposition, manifold learning and clustering - Fix for uninformative error in :class:`decomposition.IncrementalPCA`: