From 70b7af59239d9efa95b31aed1a4f89b852fe1c20 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Sun, 9 May 2021 21:26:58 +0200 Subject: [PATCH 1/7] iter --- sklearn/gaussian_process/kernels.py | 87 ++++++++++++++----- .../gaussian_process/tests/test_kernels.py | 21 +++++ 2 files changed, 88 insertions(+), 20 deletions(-) diff --git a/sklearn/gaussian_process/kernels.py b/sklearn/gaussian_process/kernels.py index 008c24f294737..68744c84b7889 100644 --- a/sklearn/gaussian_process/kernels.py +++ b/sklearn/gaussian_process/kernels.py @@ -1975,37 +1975,84 @@ def __call__(self, X, Y=None, eval_gradient=False): is True. """ X = np.atleast_2d(X) + periodic_cst = np.pi / self.periodicity + if Y is None: - dists = squareform(pdist(X, metric='euclidean')) - arg = np.pi * dists / self.periodicity - sin_of_arg = np.sin(arg) - K = np.exp(- 2 * (sin_of_arg / self.length_scale) ** 2) + # dists = squareform(pdist(X, metric='euclidean')) + # arg = np.pi * dists / self.periodicity + # sin_of_arg = np.sin(arg) + # K = np.exp(- 2 * (sin_of_arg / self.length_scale) ** 2) + dists = squareform(pdist( + X, + metric=lambda u, v: + np.sum(np.sin(periodic_cst * np.abs(u - v)) ** 2) + )) + K = np.exp(-2 * self.length_scale ** 2 * dists) else: if eval_gradient: raise ValueError( "Gradient can only be evaluated when Y is None.") - dists = cdist(X, Y, metric='euclidean') - K = np.exp(- 2 * (np.sin(np.pi / self.periodicity * dists) - / self.length_scale) ** 2) + # dists = cdist(X, Y, metric='euclidean') + # K = np.exp(- 2 * (np.sin(np.pi / self.periodicity * dists) + # / self.length_scale) ** 2) + dists = cdist( + X, Y, + metric=lambda u, v: + np.sum(np.sin(periodic_cst * np.abs(u - v)) ** 2) + ) + K = np.exp(-2 * self.length_scale ** 2 * dists) if eval_gradient: - cos_of_arg = np.cos(arg) - # gradient with respect to length_scale if not self.hyperparameter_length_scale.fixed: - length_scale_gradient = \ - 4 / self.length_scale**2 * sin_of_arg**2 * K - length_scale_gradient = length_scale_gradient[:, :, np.newaxis] - else: # length_scale is kept fixed + length_scale_gradient = squareform(pdist( + X, + metric=lambda u, v: + np.sum(np.sin(periodic_cst * np.abs(u - v)) ** 2) + )) + length_scale_gradient *= K + length_scale_gradient *= 4 + length_scale_gradient /= self.length_scale ** 3 + else: length_scale_gradient = np.empty((K.shape[0], K.shape[1], 0)) - # gradient with respect to p + if not self.hyperparameter_periodicity.fixed: - periodicity_gradient = \ - 4 * arg / self.length_scale**2 * cos_of_arg \ - * sin_of_arg * K - periodicity_gradient = periodicity_gradient[:, :, np.newaxis] - else: # p is kept fixed + periodicity_gradient = 4 * np.pi / ( + self.length_scale ** 2 * self.periodicity ** 2 + ) + periodicity_gradient *= squareform( + pdist(X, metric="cityblock") + ) + periodicity_gradient *= K + periodicity_gradient *= squareform(pdist( + X, + metric=lambda u, v: + np.sum(np.sin(periodic_cst * np.abs(u - v))) + )) + periodicity_gradient *= squareform(pdist( + X, + metric=lambda u, v: + np.sum(np.cos(periodic_cst * np.abs(u - v))) + )) + else: periodicity_gradient = np.empty((K.shape[0], K.shape[1], 0)) - + # cos_of_arg = np.cos(arg) + # # gradient with respect to length_scale + # if not self.hyperparameter_length_scale.fixed: + # length_scale_gradient = \ + # 4 / self.length_scale**2 * sin_of_arg**2 * K + # length_scale_gradient = length_scale_gradient[:, :, np.newaxis] + # else: # length_scale is kept fixed + # length_scale_gradient = np.empty((K.shape[0], K.shape[1], 0)) + # # gradient with respect to p + # if not self.hyperparameter_periodicity.fixed: + # periodicity_gradient = \ + # 4 * arg / self.length_scale**2 * cos_of_arg \ + # * sin_of_arg * K + # periodicity_gradient = periodicity_gradient[:, :, np.newaxis] + # else: # p is kept fixed + # periodicity_gradient = np.empty((K.shape[0], K.shape[1], 0)) + + print(self.length_scale, self.periodicity) return K, np.dstack((length_scale_gradient, periodicity_gradient)) else: return K diff --git a/sklearn/gaussian_process/tests/test_kernels.py b/sklearn/gaussian_process/tests/test_kernels.py index b56c0b06b5fc0..6c33fd7533a6f 100644 --- a/sklearn/gaussian_process/tests/test_kernels.py +++ b/sklearn/gaussian_process/tests/test_kernels.py @@ -367,3 +367,24 @@ def test_rational_quadratic_kernel(): ) with pytest.raises(AttributeError, match=message): kernel(X) + + +def test_xxx(): + import numpy as np + import scipy + + L = 1.0 + + # create some train/test data on a grid + train_len = 4 + r = np.linspace(0, L, train_len) + train_x, train_y = np.meshgrid(r, r) + train_in = np.stack((train_x.flatten(), train_y.flatten()), axis=-1) + + # get the kernel + kernel = ExpSineSquared() + + K = kernel(train_in) + + print(np.sort(np.linalg.eigvals(K))) + print() From c4fe8af7cdf709805d6ceaad441ee992212e35c1 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Sun, 9 May 2021 23:37:06 +0200 Subject: [PATCH 2/7] iter --- .../gaussian_process/plot_compare_gpr_krr.py | 49 ++++++++++--------- sklearn/gaussian_process/kernels.py | 34 ++++++------- .../gaussian_process/tests/test_kernels.py | 28 +++++------ 3 files changed, 53 insertions(+), 58 deletions(-) diff --git a/examples/gaussian_process/plot_compare_gpr_krr.py b/examples/gaussian_process/plot_compare_gpr_krr.py index 1eb771673b0d6..1f93eb4f6d5b7 100644 --- a/examples/gaussian_process/plot_compare_gpr_krr.py +++ b/examples/gaussian_process/plot_compare_gpr_krr.py @@ -56,8 +56,8 @@ import matplotlib.pyplot as plt -from sklearn.kernel_ridge import KernelRidge -from sklearn.model_selection import GridSearchCV +# from sklearn.kernel_ridge import KernelRidge +# from sklearn.model_selection import GridSearchCV from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import WhiteKernel, ExpSineSquared @@ -68,28 +68,31 @@ y = np.sin(X).ravel() y += 3 * (0.5 - rng.rand(X.shape[0])) # add noise -# Fit KernelRidge with parameter selection based on 5-fold cross validation -param_grid = {"alpha": [1e0, 1e-1, 1e-2, 1e-3], - "kernel": [ExpSineSquared(l, p) - for l in np.logspace(-2, 2, 10) - for p in np.logspace(0, 2, 10)]} -kr = GridSearchCV(KernelRidge(), param_grid=param_grid) -stime = time.time() -kr.fit(X, y) -print("Time for KRR fitting: %.3f" % (time.time() - stime)) - -gp_kernel = ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) \ +# # Fit KernelRidge with parameter selection based on 5-fold cross validation +# param_grid = {"alpha": [1e0, 1e-1, 1e-2, 1e-3], +# "kernel": [ExpSineSquared(l, p) +# for l in np.logspace(-2, 2, 10) +# for p in np.logspace(0, 2, 10)]} +# kr = GridSearchCV(KernelRidge(), param_grid=param_grid) +# stime = time.time() +# kr.fit(X, y) +# print("Time for KRR fitting: %.3f" % (time.time() - stime)) + +gp_kernel = ( + 1.0 + * ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e2)) + WhiteKernel(1e-1) -gpr = GaussianProcessRegressor(kernel=gp_kernel) +) +gpr = GaussianProcessRegressor(alpha=0.5, kernel=gp_kernel, normalize_y=False) stime = time.time() gpr.fit(X, y) print("Time for GPR fitting: %.3f" % (time.time() - stime)) -# Predict using kernel ridge -X_plot = np.linspace(0, 20, 10000)[:, None] -stime = time.time() -y_kr = kr.predict(X_plot) -print("Time for KRR prediction: %.3f" % (time.time() - stime)) +# # Predict using kernel ridge +X_plot = np.linspace(0, 50, 10000)[:, None] +# stime = time.time() +# y_kr = kr.predict(X_plot) +# print("Time for KRR prediction: %.3f" % (time.time() - stime)) # Predict using gaussian process regressor stime = time.time() @@ -106,16 +109,16 @@ lw = 2 plt.scatter(X, y, c='k', label='data') plt.plot(X_plot, np.sin(X_plot), color='navy', lw=lw, label='True') -plt.plot(X_plot, y_kr, color='turquoise', lw=lw, - label='KRR (%s)' % kr.best_params_) +# plt.plot(X_plot, y_kr, color='turquoise', lw=lw, +# label='KRR (%s)' % kr.best_params_) plt.plot(X_plot, y_gpr, color='darkorange', lw=lw, label='GPR (%s)' % gpr.kernel_) plt.fill_between(X_plot[:, 0], y_gpr - y_std, y_gpr + y_std, color='darkorange', alpha=0.2) plt.xlabel('data') plt.ylabel('target') -plt.xlim(0, 20) +plt.xlim(0, 50) plt.ylim(-4, 4) plt.title('GPR versus Kernel Ridge') -plt.legend(loc="best", scatterpoints=1, prop={'size': 8}) +plt.legend(loc="best", scatterpoints=1, prop={'size': 8}) plt.show() diff --git a/sklearn/gaussian_process/kernels.py b/sklearn/gaussian_process/kernels.py index 68744c84b7889..6add232886ac4 100644 --- a/sklearn/gaussian_process/kernels.py +++ b/sklearn/gaussian_process/kernels.py @@ -1985,9 +1985,9 @@ def __call__(self, X, Y=None, eval_gradient=False): dists = squareform(pdist( X, metric=lambda u, v: - np.sum(np.sin(periodic_cst * np.abs(u - v)) ** 2) + np.sum(np.sin(periodic_cst * (u - v)) ** 2) )) - K = np.exp(-2 * self.length_scale ** 2 * dists) + K = np.exp(-2.0 / self.length_scale ** 2 * dists) else: if eval_gradient: raise ValueError( @@ -1998,41 +1998,38 @@ def __call__(self, X, Y=None, eval_gradient=False): dists = cdist( X, Y, metric=lambda u, v: - np.sum(np.sin(periodic_cst * np.abs(u - v)) ** 2) + np.sum(np.sin(periodic_cst * (u - v)) ** 2) ) - K = np.exp(-2 * self.length_scale ** 2 * dists) + K = np.exp(-2.0 / self.length_scale ** 2 * dists) if eval_gradient: if not self.hyperparameter_length_scale.fixed: - length_scale_gradient = squareform(pdist( - X, - metric=lambda u, v: - np.sum(np.sin(periodic_cst * np.abs(u - v)) ** 2) - )) - length_scale_gradient *= K - length_scale_gradient *= 4 - length_scale_gradient /= self.length_scale ** 3 + length_scale_gradient = 4 * K / self.length_scale ** 3 + length_scale_gradient *= dists + length_scale_gradient = length_scale_gradient[..., np.newaxis] else: length_scale_gradient = np.empty((K.shape[0], K.shape[1], 0)) if not self.hyperparameter_periodicity.fixed: - periodicity_gradient = 4 * np.pi / ( + periodicity_gradient = 4 * np.pi * squareform(pdist( + X, + metric=lambda u, v: np.sum(u - v) + )) + periodicity_gradient /= ( self.length_scale ** 2 * self.periodicity ** 2 ) - periodicity_gradient *= squareform( - pdist(X, metric="cityblock") - ) periodicity_gradient *= K periodicity_gradient *= squareform(pdist( X, metric=lambda u, v: - np.sum(np.sin(periodic_cst * np.abs(u - v))) + np.sum(np.sin(periodic_cst * (u - v))) )) periodicity_gradient *= squareform(pdist( X, metric=lambda u, v: - np.sum(np.cos(periodic_cst * np.abs(u - v))) + np.sum(np.cos(periodic_cst * (u - v))) )) + periodicity_gradient = periodicity_gradient[..., np.newaxis] else: periodicity_gradient = np.empty((K.shape[0], K.shape[1], 0)) # cos_of_arg = np.cos(arg) @@ -2052,7 +2049,6 @@ def __call__(self, X, Y=None, eval_gradient=False): # else: # p is kept fixed # periodicity_gradient = np.empty((K.shape[0], K.shape[1], 0)) - print(self.length_scale, self.periodicity) return K, np.dstack((length_scale_gradient, periodicity_gradient)) else: return K diff --git a/sklearn/gaussian_process/tests/test_kernels.py b/sklearn/gaussian_process/tests/test_kernels.py index 6c33fd7533a6f..c1fe069382f4e 100644 --- a/sklearn/gaussian_process/tests/test_kernels.py +++ b/sklearn/gaussian_process/tests/test_kernels.py @@ -370,21 +370,17 @@ def test_rational_quadratic_kernel(): def test_xxx(): - import numpy as np - import scipy + from sklearn.gaussian_process import GaussianProcessRegressor + rng = np.random.RandomState(0) - L = 1.0 + # Generate sample data + X = 15 * rng.rand(10, 1) + y = np.sin(X).ravel() + y += 3 * (0.5 - rng.rand(X.shape[0])) # add noise - # create some train/test data on a grid - train_len = 4 - r = np.linspace(0, L, train_len) - train_x, train_y = np.meshgrid(r, r) - train_in = np.stack((train_x.flatten(), train_y.flatten()), axis=-1) - - # get the kernel - kernel = ExpSineSquared() - - K = kernel(train_in) - - print(np.sort(np.linalg.eigvals(K))) - print() + gp_kernel = ( + ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) + + WhiteKernel(1e-1) + ) + gpr = GaussianProcessRegressor(kernel=gp_kernel, normalize_y=True) + gpr.fit(X, y) From c3667e80c3d9e4d0525d24c8611b232021024f98 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Sun, 9 May 2021 23:49:43 +0200 Subject: [PATCH 3/7] iter --- sklearn/gaussian_process/kernels.py | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/sklearn/gaussian_process/kernels.py b/sklearn/gaussian_process/kernels.py index 6add232886ac4..e497b29abc466 100644 --- a/sklearn/gaussian_process/kernels.py +++ b/sklearn/gaussian_process/kernels.py @@ -1985,7 +1985,7 @@ def __call__(self, X, Y=None, eval_gradient=False): dists = squareform(pdist( X, metric=lambda u, v: - np.sum(np.sin(periodic_cst * (u - v)) ** 2) + np.sum(np.sin(periodic_cst * np.abs(u - v)) ** 2) )) K = np.exp(-2.0 / self.length_scale ** 2 * dists) else: @@ -1998,7 +1998,7 @@ def __call__(self, X, Y=None, eval_gradient=False): dists = cdist( X, Y, metric=lambda u, v: - np.sum(np.sin(periodic_cst * (u - v)) ** 2) + np.sum(np.sin(periodic_cst * np.abs(u - v)) ** 2) ) K = np.exp(-2.0 / self.length_scale ** 2 * dists) @@ -2013,7 +2013,8 @@ def __call__(self, X, Y=None, eval_gradient=False): if not self.hyperparameter_periodicity.fixed: periodicity_gradient = 4 * np.pi * squareform(pdist( X, - metric=lambda u, v: np.sum(u - v) + # metric=lambda u, v: np.sum(np.abs(u - v)) + metric="cityblock" )) periodicity_gradient /= ( self.length_scale ** 2 * self.periodicity ** 2 @@ -2022,32 +2023,16 @@ def __call__(self, X, Y=None, eval_gradient=False): periodicity_gradient *= squareform(pdist( X, metric=lambda u, v: - np.sum(np.sin(periodic_cst * (u - v))) + np.sum(np.sin(periodic_cst * np.abs(u - v))) )) periodicity_gradient *= squareform(pdist( X, metric=lambda u, v: - np.sum(np.cos(periodic_cst * (u - v))) + np.sum(np.cos(periodic_cst * np.abs(u - v))) )) periodicity_gradient = periodicity_gradient[..., np.newaxis] else: periodicity_gradient = np.empty((K.shape[0], K.shape[1], 0)) - # cos_of_arg = np.cos(arg) - # # gradient with respect to length_scale - # if not self.hyperparameter_length_scale.fixed: - # length_scale_gradient = \ - # 4 / self.length_scale**2 * sin_of_arg**2 * K - # length_scale_gradient = length_scale_gradient[:, :, np.newaxis] - # else: # length_scale is kept fixed - # length_scale_gradient = np.empty((K.shape[0], K.shape[1], 0)) - # # gradient with respect to p - # if not self.hyperparameter_periodicity.fixed: - # periodicity_gradient = \ - # 4 * arg / self.length_scale**2 * cos_of_arg \ - # * sin_of_arg * K - # periodicity_gradient = periodicity_gradient[:, :, np.newaxis] - # else: # p is kept fixed - # periodicity_gradient = np.empty((K.shape[0], K.shape[1], 0)) return K, np.dstack((length_scale_gradient, periodicity_gradient)) else: From f45782cd6830fdf1a24752ef4784f5f86f79e6f3 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Mon, 10 May 2021 00:47:37 +0200 Subject: [PATCH 4/7] iter --- .../gaussian_process/plot_compare_gpr_krr.py | 2 +- sklearn/gaussian_process/kernels.py | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/examples/gaussian_process/plot_compare_gpr_krr.py b/examples/gaussian_process/plot_compare_gpr_krr.py index 1f93eb4f6d5b7..049324249c929 100644 --- a/examples/gaussian_process/plot_compare_gpr_krr.py +++ b/examples/gaussian_process/plot_compare_gpr_krr.py @@ -83,7 +83,7 @@ * ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e2)) + WhiteKernel(1e-1) ) -gpr = GaussianProcessRegressor(alpha=0.5, kernel=gp_kernel, normalize_y=False) +gpr = GaussianProcessRegressor(alpha=0.5, kernel=gp_kernel, normalize_y=True) stime = time.time() gpr.fit(X, y) print("Time for GPR fitting: %.3f" % (time.time() - stime)) diff --git a/sklearn/gaussian_process/kernels.py b/sklearn/gaussian_process/kernels.py index e497b29abc466..0ecf03fb0d093 100644 --- a/sklearn/gaussian_process/kernels.py +++ b/sklearn/gaussian_process/kernels.py @@ -1987,7 +1987,7 @@ def __call__(self, X, Y=None, eval_gradient=False): metric=lambda u, v: np.sum(np.sin(periodic_cst * np.abs(u - v)) ** 2) )) - K = np.exp(-2.0 / self.length_scale ** 2 * dists) + K = np.exp(-2 * dists / self.length_scale ** 2) else: if eval_gradient: raise ValueError( @@ -2000,25 +2000,24 @@ def __call__(self, X, Y=None, eval_gradient=False): metric=lambda u, v: np.sum(np.sin(periodic_cst * np.abs(u - v)) ** 2) ) - K = np.exp(-2.0 / self.length_scale ** 2 * dists) + K = np.exp(-2 * dists / self.length_scale ** 2) if eval_gradient: if not self.hyperparameter_length_scale.fixed: - length_scale_gradient = 4 * K / self.length_scale ** 3 + length_scale_gradient = 4 / self.length_scale ** 3 + length_scale_gradient *= K length_scale_gradient *= dists length_scale_gradient = length_scale_gradient[..., np.newaxis] else: length_scale_gradient = np.empty((K.shape[0], K.shape[1], 0)) if not self.hyperparameter_periodicity.fixed: - periodicity_gradient = 4 * np.pi * squareform(pdist( - X, - # metric=lambda u, v: np.sum(np.abs(u - v)) - metric="cityblock" - )) - periodicity_gradient /= ( + periodicity_gradient = 4 / ( self.length_scale ** 2 * self.periodicity ** 2 ) + periodicity_gradient *= squareform(pdist( + X, metric=lambda u, v: np.sum(np.pi * np.abs(u - v)) + )) periodicity_gradient *= K periodicity_gradient *= squareform(pdist( X, From 4419e672dfb888d441679e83f51cdb7952e42a4e Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Mon, 10 May 2021 11:54:56 +0200 Subject: [PATCH 5/7] iter --- sklearn/gaussian_process/kernels.py | 45 +++++++++++-------- .../gaussian_process/tests/test_kernels.py | 21 +++++++++ 2 files changed, 48 insertions(+), 18 deletions(-) diff --git a/sklearn/gaussian_process/kernels.py b/sklearn/gaussian_process/kernels.py index 0ecf03fb0d093..e72e95c993d08 100644 --- a/sklearn/gaussian_process/kernels.py +++ b/sklearn/gaussian_process/kernels.py @@ -1982,10 +1982,12 @@ def __call__(self, X, Y=None, eval_gradient=False): # arg = np.pi * dists / self.periodicity # sin_of_arg = np.sin(arg) # K = np.exp(- 2 * (sin_of_arg / self.length_scale) ** 2) + + # K = exp(-2/l^2 * sum_i( sin^2 (pi/p * (x_i - x_i')) )) dists = squareform(pdist( X, metric=lambda u, v: - np.sum(np.sin(periodic_cst * np.abs(u - v)) ** 2) + np.sum(np.sin(periodic_cst * (u - v)) ** 2) )) K = np.exp(-2 * dists / self.length_scale ** 2) else: @@ -1995,16 +1997,20 @@ def __call__(self, X, Y=None, eval_gradient=False): # dists = cdist(X, Y, metric='euclidean') # K = np.exp(- 2 * (np.sin(np.pi / self.periodicity * dists) # / self.length_scale) ** 2) + + # K = exp(-2/l^2 * sum_i( sin^2 (pi/p * (x_i - y_i)) )) dists = cdist( X, Y, metric=lambda u, v: - np.sum(np.sin(periodic_cst * np.abs(u - v)) ** 2) + np.sum(np.sin(periodic_cst * (u - v)) ** 2) ) K = np.exp(-2 * dists / self.length_scale ** 2) if eval_gradient: if not self.hyperparameter_length_scale.fixed: - length_scale_gradient = 4 / self.length_scale ** 3 + # dK/dl = 4/l^3 * K * sum_i( sin^2 (pi/p * (x_i - x_i')) ) + # approximation is taking 4/l^2 instead of + length_scale_gradient = 2 / self.length_scale ** 3 length_scale_gradient *= K length_scale_gradient *= dists length_scale_gradient = length_scale_gradient[..., np.newaxis] @@ -2012,23 +2018,26 @@ def __call__(self, X, Y=None, eval_gradient=False): length_scale_gradient = np.empty((K.shape[0], K.shape[1], 0)) if not self.hyperparameter_periodicity.fixed: - periodicity_gradient = 4 / ( - self.length_scale ** 2 * self.periodicity ** 2 + # dK/dp = (4 * pi)/(l^2 * p^2) * sum_i( + # sin(pi/p * (x_i - x_i')) * + # cos(pi/p * (x_i - x_i')) * + # (x_i - x_i') + # ) + # approximation is taking p instead of p^2 + periodicity_gradient = (4 * np.pi) / ( + self.length_scale ** 2 * self.periodicity ) - periodicity_gradient *= squareform(pdist( - X, metric=lambda u, v: np.sum(np.pi * np.abs(u - v)) - )) periodicity_gradient *= K - periodicity_gradient *= squareform(pdist( - X, - metric=lambda u, v: - np.sum(np.sin(periodic_cst * np.abs(u - v))) - )) - periodicity_gradient *= squareform(pdist( - X, - metric=lambda u, v: - np.sum(np.cos(periodic_cst * np.abs(u - v))) - )) + periodicity_gradient *= squareform( + pdist( + X, + metric=lambda u, v: np.sum( + np.sin(periodic_cst * (u - v)) + * np.cos(periodic_cst * (u - v)) + * (u - v) + ), + ) + ) periodicity_gradient = periodicity_gradient[..., np.newaxis] else: periodicity_gradient = np.empty((K.shape[0], K.shape[1], 0)) diff --git a/sklearn/gaussian_process/tests/test_kernels.py b/sklearn/gaussian_process/tests/test_kernels.py index c1fe069382f4e..82090e9d87f62 100644 --- a/sklearn/gaussian_process/tests/test_kernels.py +++ b/sklearn/gaussian_process/tests/test_kernels.py @@ -384,3 +384,24 @@ def test_xxx(): ) gpr = GaussianProcessRegressor(kernel=gp_kernel, normalize_y=True) gpr.fit(X, y) + + +def test_yyy(): + import numpy as np + from sklearn.gaussian_process.kernels import ExpSineSquared + + L = 1.0 + + # create some train/test data on a grid + train_len = 4 + r = np.linspace(0, L, train_len) + train_x, train_y = np.meshgrid(r, r) + train_in = np.stack((train_x.flatten(), train_y.flatten()), axis=-1) + + # get the kernel + kernel = ExpSineSquared() + + K = kernel(train_in) + 1e-4 * np.eye(train_len**2) + + print(np.sort(np.linalg.eigh(K)[0])) + print() From 33d018f1dd8d290731fb18f81fe7d40e4ab48ff0 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Mon, 10 May 2021 19:25:00 +0200 Subject: [PATCH 6/7] iter --- examples/gaussian_process/plot_compare_gpr_krr.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/gaussian_process/plot_compare_gpr_krr.py b/examples/gaussian_process/plot_compare_gpr_krr.py index 049324249c929..93af9f9a8aa7a 100644 --- a/examples/gaussian_process/plot_compare_gpr_krr.py +++ b/examples/gaussian_process/plot_compare_gpr_krr.py @@ -59,14 +59,14 @@ # from sklearn.kernel_ridge import KernelRidge # from sklearn.model_selection import GridSearchCV from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.gaussian_process.kernels import WhiteKernel, ExpSineSquared +from sklearn.gaussian_process.kernels import WhiteKernel, ExpSineSquared, RBF rng = np.random.RandomState(0) # Generate sample data X = 15 * rng.rand(100, 1) y = np.sin(X).ravel() -y += 3 * (0.5 - rng.rand(X.shape[0])) # add noise +y += 1.5 * (0.5 - rng.rand(X.shape[0])) # add noise # # Fit KernelRidge with parameter selection based on 5-fold cross validation # param_grid = {"alpha": [1e0, 1e-1, 1e-2, 1e-3], @@ -79,17 +79,17 @@ # print("Time for KRR fitting: %.3f" % (time.time() - stime)) gp_kernel = ( - 1.0 - * ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e2)) + ExpSineSquared(1.0, periodicity=5.0, periodicity_bounds=(1e-2, 1e1)) + * RBF(length_scale=10, length_scale_bounds=(1e-2, 1e2)) + WhiteKernel(1e-1) ) -gpr = GaussianProcessRegressor(alpha=0.5, kernel=gp_kernel, normalize_y=True) +gpr = GaussianProcessRegressor(kernel=gp_kernel, normalize_y=True) stime = time.time() gpr.fit(X, y) print("Time for GPR fitting: %.3f" % (time.time() - stime)) # # Predict using kernel ridge -X_plot = np.linspace(0, 50, 10000)[:, None] +X_plot = np.linspace(0, 40, 1000)[:, None] # stime = time.time() # y_kr = kr.predict(X_plot) # print("Time for KRR prediction: %.3f" % (time.time() - stime)) @@ -117,7 +117,7 @@ alpha=0.2) plt.xlabel('data') plt.ylabel('target') -plt.xlim(0, 50) +plt.xlim(0, 40) plt.ylim(-4, 4) plt.title('GPR versus Kernel Ridge') plt.legend(loc="best", scatterpoints=1, prop={'size': 8}) From 07ac279288a2234bd56a087cb4ab0479ff7634cc Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Mon, 17 May 2021 10:55:27 +0200 Subject: [PATCH 7/7] iter --- sklearn/gaussian_process/kernels.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/sklearn/gaussian_process/kernels.py b/sklearn/gaussian_process/kernels.py index e72e95c993d08..4d77255c2471d 100644 --- a/sklearn/gaussian_process/kernels.py +++ b/sklearn/gaussian_process/kernels.py @@ -1978,11 +1978,6 @@ def __call__(self, X, Y=None, eval_gradient=False): periodic_cst = np.pi / self.periodicity if Y is None: - # dists = squareform(pdist(X, metric='euclidean')) - # arg = np.pi * dists / self.periodicity - # sin_of_arg = np.sin(arg) - # K = np.exp(- 2 * (sin_of_arg / self.length_scale) ** 2) - # K = exp(-2/l^2 * sum_i( sin^2 (pi/p * (x_i - x_i')) )) dists = squareform(pdist( X, @@ -1994,10 +1989,6 @@ def __call__(self, X, Y=None, eval_gradient=False): if eval_gradient: raise ValueError( "Gradient can only be evaluated when Y is None.") - # dists = cdist(X, Y, metric='euclidean') - # K = np.exp(- 2 * (np.sin(np.pi / self.periodicity * dists) - # / self.length_scale) ** 2) - # K = exp(-2/l^2 * sum_i( sin^2 (pi/p * (x_i - y_i)) )) dists = cdist( X, Y, @@ -2009,8 +2000,7 @@ def __call__(self, X, Y=None, eval_gradient=False): if eval_gradient: if not self.hyperparameter_length_scale.fixed: # dK/dl = 4/l^3 * K * sum_i( sin^2 (pi/p * (x_i - x_i')) ) - # approximation is taking 4/l^2 instead of - length_scale_gradient = 2 / self.length_scale ** 3 + length_scale_gradient = 4 / self.length_scale ** 3 length_scale_gradient *= K length_scale_gradient *= dists length_scale_gradient = length_scale_gradient[..., np.newaxis] @@ -2023,9 +2013,8 @@ def __call__(self, X, Y=None, eval_gradient=False): # cos(pi/p * (x_i - x_i')) * # (x_i - x_i') # ) - # approximation is taking p instead of p^2 periodicity_gradient = (4 * np.pi) / ( - self.length_scale ** 2 * self.periodicity + self.length_scale ** 2 * self.periodicity ** 2 ) periodicity_gradient *= K periodicity_gradient *= squareform(