scikit-learn · glemaitre · May 9, 2021 · May 9, 2021 · May 9, 2021 · May 9, 2021
diff --git a/examples/gaussian_process/plot_compare_gpr_krr.py b/examples/gaussian_process/plot_compare_gpr_krr.py
@@ -56,40 +56,43 @@
 
 import matplotlib.pyplot as plt
 
-from sklearn.kernel_ridge import KernelRidge
-from sklearn.model_selection import GridSearchCV
+# from sklearn.kernel_ridge import KernelRidge
+# from sklearn.model_selection import GridSearchCV
 from sklearn.gaussian_process import GaussianProcessRegressor
-from sklearn.gaussian_process.kernels import WhiteKernel, ExpSineSquared
+from sklearn.gaussian_process.kernels import WhiteKernel, ExpSineSquared, RBF
 
 rng = np.random.RandomState(0)
 
 # Generate sample data
 X = 15 * rng.rand(100, 1)
 y = np.sin(X).ravel()
-y += 3 * (0.5 - rng.rand(X.shape[0]))  # add noise
-
-# Fit KernelRidge with parameter selection based on 5-fold cross validation
-param_grid = {"alpha": [1e0, 1e-1, 1e-2, 1e-3],
-              "kernel": [ExpSineSquared(l, p)
-                         for l in np.logspace(-2, 2, 10)
-                         for p in np.logspace(0, 2, 10)]}
-kr = GridSearchCV(KernelRidge(), param_grid=param_grid)
-stime = time.time()
-kr.fit(X, y)
-print("Time for KRR fitting: %.3f" % (time.time() - stime))
-
-gp_kernel = ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) \
+y += 1.5 * (0.5 - rng.rand(X.shape[0]))  # add noise
+
+# # Fit KernelRidge with parameter selection based on 5-fold cross validation
+# param_grid = {"alpha": [1e0, 1e-1, 1e-2, 1e-3],
+#               "kernel": [ExpSineSquared(l, p)
+#                          for l in np.logspace(-2, 2, 10)
+#                          for p in np.logspace(0, 2, 10)]}
+# kr = GridSearchCV(KernelRidge(), param_grid=param_grid)
+# stime = time.time()
+# kr.fit(X, y)
+# print("Time for KRR fitting: %.3f" % (time.time() - stime))
+
+gp_kernel = (
+    ExpSineSquared(1.0, periodicity=5.0, periodicity_bounds=(1e-2, 1e1))
+    * RBF(length_scale=10, length_scale_bounds=(1e-2, 1e2))
     + WhiteKernel(1e-1)
-gpr = GaussianProcessRegressor(kernel=gp_kernel)
+)
+gpr = GaussianProcessRegressor(kernel=gp_kernel, normalize_y=True)
 stime = time.time()
 gpr.fit(X, y)
 print("Time for GPR fitting: %.3f" % (time.time() - stime))
 
-# Predict using kernel ridge
-X_plot = np.linspace(0, 20, 10000)[:, None]
-stime = time.time()
-y_kr = kr.predict(X_plot)
-print("Time for KRR prediction: %.3f" % (time.time() - stime))
+# # Predict using kernel ridge
+X_plot = np.linspace(0, 40, 1000)[:, None]
+# stime = time.time()
+# y_kr = kr.predict(X_plot)
+# print("Time for KRR prediction: %.3f" % (time.time() - stime))
 
 # Predict using gaussian process regressor
 stime = time.time()
@@ -106,16 +109,16 @@
 lw = 2
 plt.scatter(X, y, c='k', label='data')
 plt.plot(X_plot, np.sin(X_plot), color='navy', lw=lw, label='True')
-plt.plot(X_plot, y_kr, color='turquoise', lw=lw,
-         label='KRR (%s)' % kr.best_params_)
+# plt.plot(X_plot, y_kr, color='turquoise', lw=lw,
+#          label='KRR (%s)' % kr.best_params_)
 plt.plot(X_plot, y_gpr, color='darkorange', lw=lw,
          label='GPR (%s)' % gpr.kernel_)
 plt.fill_between(X_plot[:, 0], y_gpr - y_std, y_gpr + y_std, color='darkorange',
                  alpha=0.2)
 plt.xlabel('data')
 plt.ylabel('target')
-plt.xlim(0, 20)
+plt.xlim(0, 40)
 plt.ylim(-4, 4)
 plt.title('GPR versus Kernel Ridge')
-plt.legend(loc="best",  scatterpoints=1, prop={'size': 8})
+plt.legend(loc="best", scatterpoints=1, prop={'size': 8})
 plt.show()
diff --git a/sklearn/gaussian_process/kernels.py b/sklearn/gaussian_process/kernels.py
@@ -1975,35 +1975,60 @@ def __call__(self, X, Y=None, eval_gradient=False):
             is True.
         """
         X = np.atleast_2d(X)
+        periodic_cst = np.pi / self.periodicity
+
         if Y is None:
-            dists = squareform(pdist(X, metric='euclidean'))
-            arg = np.pi * dists / self.periodicity
-            sin_of_arg = np.sin(arg)
-            K = np.exp(- 2 * (sin_of_arg / self.length_scale) ** 2)
+            # K = exp(-2/l^2 * sum_i( sin^2 (pi/p * (x_i - x_i')) ))
+            dists = squareform(pdist(
+                X,
+                metric=lambda u, v:
+                np.sum(np.sin(periodic_cst * (u - v)) ** 2)
+            ))
+            K = np.exp(-2 * dists / self.length_scale ** 2)
         else:
             if eval_gradient:
                 raise ValueError(
                     "Gradient can only be evaluated when Y is None.")
-            dists = cdist(X, Y, metric='euclidean')
-            K = np.exp(- 2 * (np.sin(np.pi / self.periodicity * dists)
-                              / self.length_scale) ** 2)
+            # K = exp(-2/l^2 * sum_i( sin^2 (pi/p * (x_i - y_i)) ))
+            dists = cdist(
+                X, Y,
+                metric=lambda u, v:
+                np.sum(np.sin(periodic_cst * (u - v)) ** 2)
+            )
+            K = np.exp(-2 * dists / self.length_scale ** 2)
 
         if eval_gradient:
-            cos_of_arg = np.cos(arg)
-            # gradient with respect to length_scale
             if not self.hyperparameter_length_scale.fixed:
-                length_scale_gradient = \
-                    4 / self.length_scale**2 * sin_of_arg**2 * K
-                length_scale_gradient = length_scale_gradient[:, :, np.newaxis]
-            else:  # length_scale is kept fixed
+                # dK/dl = 4/l^3 * K * sum_i( sin^2 (pi/p * (x_i - x_i')) )
+                length_scale_gradient = 4 / self.length_scale ** 3
+                length_scale_gradient *= K
+                length_scale_gradient *= dists
+                length_scale_gradient = length_scale_gradient[..., np.newaxis]
+            else:
                 length_scale_gradient = np.empty((K.shape[0], K.shape[1], 0))
-            # gradient with respect to p
+
             if not self.hyperparameter_periodicity.fixed:
-                periodicity_gradient = \
-                    4 * arg / self.length_scale**2 * cos_of_arg \
-                    * sin_of_arg * K
-                periodicity_gradient = periodicity_gradient[:, :, np.newaxis]
-            else:  # p is kept fixed
+                # dK/dp = (4 * pi)/(l^2 * p^2) * sum_i(
+                #    sin(pi/p * (x_i - x_i')) *
+                #    cos(pi/p * (x_i - x_i')) *
+                #    (x_i - x_i')
+                # )
+                periodicity_gradient = (4 * np.pi) / (
+                    self.length_scale ** 2 * self.periodicity ** 2
+                )
+                periodicity_gradient *= K
+                periodicity_gradient *= squareform(
+                    pdist(
+                        X,
+                        metric=lambda u, v: np.sum(
+                            np.sin(periodic_cst * (u - v))
+                            * np.cos(periodic_cst * (u - v))
+                            * (u - v)
+                        ),
+                    )
+                )
+                periodicity_gradient = periodicity_gradient[..., np.newaxis]
+            else:
                 periodicity_gradient = np.empty((K.shape[0], K.shape[1], 0))
 
             return K, np.dstack((length_scale_gradient, periodicity_gradient))

diff --git a/sklearn/gaussian_process/tests/test_kernels.py b/sklearn/gaussian_process/tests/test_kernels.py
@@ -367,3 +367,41 @@ def test_rational_quadratic_kernel():
     )
     with pytest.raises(AttributeError, match=message):
         kernel(X)
+
+
+def test_xxx():
+    from sklearn.gaussian_process import GaussianProcessRegressor
+    rng = np.random.RandomState(0)
+
+    # Generate sample data
+    X = 15 * rng.rand(10, 1)
+    y = np.sin(X).ravel()
+    y += 3 * (0.5 - rng.rand(X.shape[0]))  # add noise
+
+    gp_kernel = (
+        ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) +
+        WhiteKernel(1e-1)
+    )
+    gpr = GaussianProcessRegressor(kernel=gp_kernel, normalize_y=True)
+    gpr.fit(X, y)
+
+
+def test_yyy():
+    import numpy as np
+    from sklearn.gaussian_process.kernels import ExpSineSquared
+
+    L = 1.0
+
+    # create some train/test data on a grid
+    train_len = 4
+    r = np.linspace(0, L, train_len)
+    train_x, train_y = np.meshgrid(r, r)
+    train_in = np.stack((train_x.flatten(), train_y.flatten()), axis=-1)
+
+    # get the kernel
+    kernel = ExpSineSquared()
+
+    K = kernel(train_in) + 1e-4 * np.eye(train_len**2)
+
+    print(np.sort(np.linalg.eigh(K)[0]))
+    print()