tests with WLS

giorgiop · giorgiop · commit b2e6dcbf0081 · 2015-11-07T09:59:54.000+01:00
diff --git a/sklearn/linear_model/tests/test_base.py b/sklearn/linear_model/tests/test_base.py
@@ -5,6 +5,7 @@
 
 import numpy as np
 from scipy import sparse
+from scipy import linalg
 
 from sklearn.utils.testing import assert_array_almost_equal
 from sklearn.utils.testing import assert_almost_equal
@@ -55,25 +56,15 @@ def test_linear_regression_sample_weights():
 
         for intercept in (True, False):
 
+            # LinearRegression with explicit sample_weight
             reg = LinearRegression(fit_intercept=intercept)
             reg.fit(X, y, sample_weight=sample_weight)
             coefs1 = reg.coef_
             inter1 = reg.intercept_
 
-            assert_equal(reg.coef_.shape, (X.shape[1], ))
+            assert_equal(reg.coef_.shape, (X.shape[1], ))  # sanity checks
             assert_greater(reg.score(X, y), 0.5)
 
-            # Sample weight can be implemented via a simple rescaling
-            # for the square loss.
-            scaled_y = y * np.sqrt(sample_weight)
-            scaled_X = X * np.sqrt(sample_weight)[:, np.newaxis]
-            reg.fit(scaled_X, scaled_y)
-            coefs2 = reg.coef_
-            inter2 = reg.intercept_
-
-            # assert_array_almost_equal(coefs1, coefs2)
-            # assert_almost_equal(inter1, inter2)
-
             # Closed form of the weighted least square
             # theta = (X^T W X)^(-1) * X^T W y
             W = np.diag(sample_weight)
@@ -82,8 +73,8 @@ def test_linear_regression_sample_weights():
             else:
                 X_aug = np.column_stack((np.ones(n_samples), X))
 
-            coefs3 = np.linalg.pinv(X_aug.T.dot(W).dot(X_aug)
-                                    ).dot(X_aug.T).dot(W).dot(y)
+            coefs3 = linalg.pinv(X_aug.T.dot(W).dot(X_aug)
+                                 ).dot(X_aug.T).dot(W).dot(y)
 
             if intercept is False:
                 assert_array_almost_equal(coefs1, coefs3)
diff --git a/sklearn/linear_model/tests/test_ridge.py b/sklearn/linear_model/tests/test_ridge.py
@@ -1,6 +1,7 @@
 import numpy as np
 import scipy.sparse as sp
 from scipy import linalg
+from itertools import product
 
 from sklearn.utils.testing import assert_true
 from sklearn.utils.testing import assert_almost_equal
@@ -111,7 +112,7 @@ def test_ridge_singular():
     assert_greater(ridge.score(X, y), 0.9)
 
 
-def test_ridge_sample_weights():
+def test_ridge_regression_sample_weights():
     rng = np.random.RandomState(0)
 
     for solver in ("cholesky", ):
@@ -125,6 +126,7 @@ def test_ridge_sample_weights():
                                          alpha=alpha,
                                          sample_weight=sample_weight,
                                          solver=solver)
+
                 # Sample weight can be implemented via a simple rescaling
                 # for the square loss.
                 coefs2 = ridge_regression(
@@ -133,32 +135,46 @@ def test_ridge_sample_weights():
                     alpha=alpha, solver=solver)
                 assert_array_almost_equal(coefs, coefs2)
 
-                # Test for fit_intercept = True
-                est = Ridge(alpha=alpha, solver=solver)
-                est.fit(X, y, sample_weight=sample_weight)
-
-                # Check using Newton's Method
-                # Quadratic function should be solved in a single step.
-                # Initialize
-                sample_weight = np.sqrt(sample_weight)
-                X_weighted = sample_weight[:, np.newaxis] * (
-                    np.column_stack((np.ones(n_samples), X)))
-                y_weighted = y * sample_weight
-
-                # Gradient is (X*coef-y)*X + alpha*coef_[1:]
-                # Remove coef since it is initialized to zero.
-                grad = -np.dot(y_weighted, X_weighted)
-
-                # Hessian is (X.T*X) + alpha*I except that the first
-                # diagonal element should be zero, since there is no
-                # penalization of intercept.
-                diag = alpha * np.ones(n_features + 1)
-                diag[0] = 0.
-                hess = np.dot(X_weighted.T, X_weighted)
-                hess.flat[::n_features + 2] += diag
-                coef_ = - np.dot(linalg.inv(hess), grad)
-                assert_almost_equal(coef_[0], est.intercept_)
-                assert_array_almost_equal(coef_[1:], est.coef_)
+
+def test_ridge_sample_weights():
+    rng = np.random.RandomState(0)
+    param_grid = product((1.0, 1e-2), (True, False),
+                         ('svd', 'cholesky', 'lsqr', 'sparse_cg'))
+
+    for n_samples, n_features in ((6, 5), (5, 10)):
+
+        y = rng.randn(n_samples)
+        X = rng.randn(n_samples, n_features)
+        sample_weight = 1 + rng.rand(n_samples)
+
+        for (alpha, intercept, solver) in param_grid:
+            print(solver)
+
+            # Ridge with explicit sample_weight
+            est = Ridge(alpha=alpha, fit_intercept=intercept, solver=solver)
+            est.fit(X, y, sample_weight=sample_weight)
+            coefs = est.coef_
+            inter = est.intercept_
+
+            # Closed form of the weighted regularized least square
+            # theta = (X^T W X + alpha I)^(-1) * X^T W y
+            W = np.diag(sample_weight)
+            if intercept is False:
+                X_aug = X.copy()
+                I = np.eye(n_features)
+            else:
+                X_aug = np.column_stack((np.ones(n_samples), X))
+                I = np.eye(n_features + 1)
+                I[0, 0] = 0
+
+            cf_coefs = linalg.inv(X_aug.T.dot(W).dot(X_aug) +
+                                  alpha * I).dot(X_aug.T).dot(W).dot(y)
+
+            if intercept is False:
+                assert_array_almost_equal(coefs, cf_coefs)
+            else:
+                assert_array_almost_equal(coefs, cf_coefs[1:])
+                assert_almost_equal(inter, cf_coefs[0])
 
 
 def test_ridge_shapes():
