From 289eb919081fd15345c3ea88bdede1d544f1b021 Mon Sep 17 00:00:00 2001 From: larkee Date: Thu, 21 May 2020 14:38:02 +1000 Subject: [PATCH 1/3] feat: add support for using the emulator programatically --- google/cloud/spanner_v1/client.py | 45 +++++++++-------- google/cloud/spanner_v1/database.py | 4 +- tests/unit/test_client.py | 75 +++++++++++++++++++++++++---- 3 files changed, 92 insertions(+), 32 deletions(-) diff --git a/google/cloud/spanner_v1/client.py b/google/cloud/spanner_v1/client.py index 89ab490cff..1b7ef806b1 100644 --- a/google/cloud/spanner_v1/client.py +++ b/google/cloud/spanner_v1/client.py @@ -28,6 +28,7 @@ import warnings from google.api_core.gapic_v1 import client_info +from google.auth.credentials import AnonymousCredentials import google.api_core.client_options from google.cloud.spanner_admin_instance_v1.gapic.transports import ( @@ -173,6 +174,21 @@ def __init__( client_options=None, query_options=None, ): + self._emulator_host = _get_spanner_emulator_host() + + if client_options and type(client_options) == dict: + self._client_options = google.api_core.client_options.from_dict( + client_options + ) + else: + self._client_options = client_options + + if isinstance(credentials, AnonymousCredentials): + self._emulator_host = self._client_options.api_endpoint + + if not credentials and _get_spanner_emulator_host(): + credentials = AnonymousCredentials() + # NOTE: This API has no use for the _http argument, but sending it # will have no impact since the _http() @property only lazily # creates a working HTTP object. @@ -180,12 +196,6 @@ def __init__( project=project, credentials=credentials, _http=None ) self._client_info = client_info - if client_options and type(client_options) == dict: - self._client_options = google.api_core.client_options.from_dict( - client_options - ) - else: - self._client_options = client_options env_query_options = ExecuteSqlRequest.QueryOptions( optimizer_version=_get_spanner_optimizer_version() @@ -198,9 +208,8 @@ def __init__( warnings.warn(_USER_AGENT_DEPRECATED, DeprecationWarning, stacklevel=2) self.user_agent = user_agent - if _get_spanner_emulator_host() is not None and ( - "http://" in _get_spanner_emulator_host() - or "https://" in _get_spanner_emulator_host() + if self._emulator_host is not None and ( + "http://" in self._emulator_host or "https://" in self._emulator_host ): warnings.warn(_EMULATOR_HOST_HTTP_SCHEME) @@ -237,14 +246,12 @@ def project_name(self): def instance_admin_api(self): """Helper for session-related API calls.""" if self._instance_admin_api is None: - if _get_spanner_emulator_host() is not None: + if self._emulator_host is not None: transport = instance_admin_grpc_transport.InstanceAdminGrpcTransport( - channel=grpc.insecure_channel(_get_spanner_emulator_host()) + channel=grpc.insecure_channel(target=self._emulator_host) ) self._instance_admin_api = InstanceAdminClient( - client_info=self._client_info, - client_options=self._client_options, - transport=transport, + client_info=self._client_info, transport=transport ) else: self._instance_admin_api = InstanceAdminClient( @@ -258,14 +265,12 @@ def instance_admin_api(self): def database_admin_api(self): """Helper for session-related API calls.""" if self._database_admin_api is None: - if _get_spanner_emulator_host() is not None: + if self._emulator_host is not None: transport = database_admin_grpc_transport.DatabaseAdminGrpcTransport( - channel=grpc.insecure_channel(_get_spanner_emulator_host()) + channel=grpc.insecure_channel(target=self._emulator_host) ) self._database_admin_api = DatabaseAdminClient( - client_info=self._client_info, - client_options=self._client_options, - transport=transport, + client_info=self._client_info, transport=transport ) else: self._database_admin_api = DatabaseAdminClient( @@ -363,7 +368,7 @@ def instance( configuration_name, node_count, display_name, - _get_spanner_emulator_host(), + self._emulator_host, ) def list_instances(self, filter_="", page_size=None, page_token=None): diff --git a/google/cloud/spanner_v1/database.py b/google/cloud/spanner_v1/database.py index e7f6de3724..8ece803847 100644 --- a/google/cloud/spanner_v1/database.py +++ b/google/cloud/spanner_v1/database.py @@ -223,9 +223,7 @@ def spanner_api(self): channel=grpc.insecure_channel(self._instance.emulator_host) ) self._spanner_api = SpannerClient( - client_info=client_info, - client_options=client_options, - transport=transport, + client_info=client_info, transport=transport ) return self._spanner_api credentials = self._instance._client.credentials diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index b9446fd867..ece6bee21f 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -219,6 +219,8 @@ def test_constructor_custom_query_options_env_config(self, mock_ver): def test_instance_admin_api(self, mock_em): from google.cloud.spanner_v1.client import SPANNER_ADMIN_SCOPE + mock_em.return_value = None + credentials = _make_credentials() client_info = mock.Mock() client_options = mock.Mock() @@ -230,7 +232,6 @@ def test_instance_admin_api(self, mock_em): ) expected_scopes = (SPANNER_ADMIN_SCOPE,) - mock_em.return_value = None inst_module = "google.cloud.spanner_v1.client.InstanceAdminClient" with mock.patch(inst_module) as instance_admin_client: api = client.instance_admin_api @@ -250,10 +251,37 @@ def test_instance_admin_api(self, mock_em): credentials.with_scopes.assert_called_once_with(expected_scopes) @mock.patch("google.cloud.spanner_v1.client._get_spanner_emulator_host") - def test_instance_admin_api_emulator(self, mock_em): + def test_instance_admin_api_emulator_env(self, mock_em): + mock_em.return_value = "emulator.host" credentials = _make_credentials() client_info = mock.Mock() - client_options = mock.Mock() + client = self._make_one( + project=self.PROJECT, credentials=credentials, client_info=client_info + ) + + inst_module = "google.cloud.spanner_v1.client.InstanceAdminClient" + with mock.patch(inst_module) as instance_admin_client: + api = client.instance_admin_api + + self.assertIs(api, instance_admin_client.return_value) + + # API instance is cached + again = client.instance_admin_api + self.assertIs(again, api) + + self.assertEqual(len(instance_admin_client.call_args_list), 1) + called_args, called_kw = instance_admin_client.call_args + self.assertEqual(called_args, ()) + self.assertEqual(called_kw["client_info"], client_info) + self.assertIn("transport", called_kw) + self.assertNotIn("credentials", called_kw) + + def test_instance_admin_api_emulator_code(self): + from google.auth.credentials import AnonymousCredentials + + credentials = AnonymousCredentials() + client_info = mock.Mock() + client_options = {"api_endpoint": "emulator.host"} client = self._make_one( project=self.PROJECT, credentials=credentials, @@ -261,7 +289,6 @@ def test_instance_admin_api_emulator(self, mock_em): client_options=client_options, ) - mock_em.return_value = "true" inst_module = "google.cloud.spanner_v1.client.InstanceAdminClient" with mock.patch(inst_module) as instance_admin_client: api = client.instance_admin_api @@ -276,7 +303,6 @@ def test_instance_admin_api_emulator(self, mock_em): called_args, called_kw = instance_admin_client.call_args self.assertEqual(called_args, ()) self.assertEqual(called_kw["client_info"], client_info) - self.assertEqual(called_kw["client_options"], client_options) self.assertIn("transport", called_kw) self.assertNotIn("credentials", called_kw) @@ -284,6 +310,7 @@ def test_instance_admin_api_emulator(self, mock_em): def test_database_admin_api(self, mock_em): from google.cloud.spanner_v1.client import SPANNER_ADMIN_SCOPE + mock_em.return_value = None credentials = _make_credentials() client_info = mock.Mock() client_options = mock.Mock() @@ -295,7 +322,6 @@ def test_database_admin_api(self, mock_em): ) expected_scopes = (SPANNER_ADMIN_SCOPE,) - mock_em.return_value = None db_module = "google.cloud.spanner_v1.client.DatabaseAdminClient" with mock.patch(db_module) as database_admin_client: api = client.database_admin_api @@ -315,7 +341,8 @@ def test_database_admin_api(self, mock_em): credentials.with_scopes.assert_called_once_with(expected_scopes) @mock.patch("google.cloud.spanner_v1.client._get_spanner_emulator_host") - def test_database_admin_api_emulator(self, mock_em): + def test_database_admin_api_emulator_env(self, mock_em): + mock_em.return_value = "host:port" credentials = _make_credentials() client_info = mock.Mock() client_options = mock.Mock() @@ -326,7 +353,6 @@ def test_database_admin_api_emulator(self, mock_em): client_options=client_options, ) - mock_em.return_value = "host:port" db_module = "google.cloud.spanner_v1.client.DatabaseAdminClient" with mock.patch(db_module) as database_admin_client: api = client.database_admin_api @@ -341,8 +367,39 @@ def test_database_admin_api_emulator(self, mock_em): called_args, called_kw = database_admin_client.call_args self.assertEqual(called_args, ()) self.assertEqual(called_kw["client_info"], client_info) - self.assertEqual(called_kw["client_options"], client_options) self.assertIn("transport", called_kw) + self.assertNotIn("client_options", called_kw) + self.assertNotIn("credentials", called_kw) + + def test_database_admin_api_emulator_code(self): + from google.auth.credentials import AnonymousCredentials + + credentials = AnonymousCredentials() + client_info = mock.Mock() + client_options = {"api_endpoint": "emulator.host"} + client = self._make_one( + project=self.PROJECT, + credentials=credentials, + client_info=client_info, + client_options=client_options, + ) + + db_module = "google.cloud.spanner_v1.client.DatabaseAdminClient" + with mock.patch(db_module) as database_admin_client: + api = client.database_admin_api + + self.assertIs(api, database_admin_client.return_value) + + # API instance is cached + again = client.database_admin_api + self.assertIs(again, api) + + self.assertEqual(len(database_admin_client.call_args_list), 1) + called_args, called_kw = database_admin_client.call_args + self.assertEqual(called_args, ()) + self.assertEqual(called_kw["client_info"], client_info) + self.assertIn("transport", called_kw) + self.assertNotIn("client_options", called_kw) self.assertNotIn("credentials", called_kw) def test_copy(self): From d25a8b426aef0bca4015d16a82f456ff8f9271db Mon Sep 17 00:00:00 2001 From: larkee Date: Mon, 25 May 2020 10:02:26 +1000 Subject: [PATCH 2/3] always set credentials when SPANNER_EMULATOR_HOST is set --- google/cloud/spanner_v1/client.py | 7 +++---- tests/unit/test_client.py | 7 +++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/google/cloud/spanner_v1/client.py b/google/cloud/spanner_v1/client.py index 1b7ef806b1..2eb6576db2 100644 --- a/google/cloud/spanner_v1/client.py +++ b/google/cloud/spanner_v1/client.py @@ -183,11 +183,10 @@ def __init__( else: self._client_options = client_options - if isinstance(credentials, AnonymousCredentials): - self._emulator_host = self._client_options.api_endpoint - - if not credentials and _get_spanner_emulator_host(): + if self._emulator_host: credentials = AnonymousCredentials() + elif isinstance(credentials, AnonymousCredentials): + self._emulator_host = self._client_options.api_endpoint # NOTE: This API has no use for the _http argument, but sending it # will have no impact since the _http() @property only lazily diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index ece6bee21f..db8cbd17df 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -110,11 +110,14 @@ def _constructor_test_helper( @mock.patch("warnings.warn") def test_constructor_emulator_host_warning(self, mock_warn, mock_em): from google.cloud.spanner_v1 import client as MUT + from google.auth.credentials import AnonymousCredentials - expected_scopes = (MUT.SPANNER_ADMIN_SCOPE,) + expected_scopes = None creds = _make_credentials() mock_em.return_value = "http://emulator.host.com" - self._constructor_test_helper(expected_scopes, creds) + with mock.patch("google.cloud.spanner_v1.client.AnonymousCredentials") as patch: + expected_creds = patch.return_value = AnonymousCredentials() + self._constructor_test_helper(expected_scopes, creds, expected_creds) mock_warn.assert_called_once_with(MUT._EMULATOR_HOST_HTTP_SCHEME) def test_constructor_default_scopes(self): From 08cf71b9707a824289e3c7c607e939ad517f99dd Mon Sep 17 00:00:00 2001 From: larkee Date: Mon, 25 May 2020 14:08:11 +1000 Subject: [PATCH 3/3] address PR comments --- google/cloud/spanner_v1/client.py | 8 ++++++-- tests/unit/test_client.py | 18 +++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/google/cloud/spanner_v1/client.py b/google/cloud/spanner_v1/client.py index 2eb6576db2..0759fcff23 100644 --- a/google/cloud/spanner_v1/client.py +++ b/google/cloud/spanner_v1/client.py @@ -250,7 +250,9 @@ def instance_admin_api(self): channel=grpc.insecure_channel(target=self._emulator_host) ) self._instance_admin_api = InstanceAdminClient( - client_info=self._client_info, transport=transport + client_info=self._client_info, + client_options=self._client_options, + transport=transport, ) else: self._instance_admin_api = InstanceAdminClient( @@ -269,7 +271,9 @@ def database_admin_api(self): channel=grpc.insecure_channel(target=self._emulator_host) ) self._database_admin_api = DatabaseAdminClient( - client_info=self._client_info, transport=transport + client_info=self._client_info, + client_options=self._client_options, + transport=transport, ) else: self._database_admin_api = DatabaseAdminClient( diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index db8cbd17df..614bf4bde6 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -258,8 +258,12 @@ def test_instance_admin_api_emulator_env(self, mock_em): mock_em.return_value = "emulator.host" credentials = _make_credentials() client_info = mock.Mock() + client_options = mock.Mock() client = self._make_one( - project=self.PROJECT, credentials=credentials, client_info=client_info + project=self.PROJECT, + credentials=credentials, + client_info=client_info, + client_options=client_options, ) inst_module = "google.cloud.spanner_v1.client.InstanceAdminClient" @@ -276,15 +280,17 @@ def test_instance_admin_api_emulator_env(self, mock_em): called_args, called_kw = instance_admin_client.call_args self.assertEqual(called_args, ()) self.assertEqual(called_kw["client_info"], client_info) + self.assertEqual(called_kw["client_options"], client_options) self.assertIn("transport", called_kw) self.assertNotIn("credentials", called_kw) def test_instance_admin_api_emulator_code(self): from google.auth.credentials import AnonymousCredentials + from google.api_core.client_options import ClientOptions credentials = AnonymousCredentials() client_info = mock.Mock() - client_options = {"api_endpoint": "emulator.host"} + client_options = ClientOptions(api_endpoint="emulator.host") client = self._make_one( project=self.PROJECT, credentials=credentials, @@ -306,6 +312,7 @@ def test_instance_admin_api_emulator_code(self): called_args, called_kw = instance_admin_client.call_args self.assertEqual(called_args, ()) self.assertEqual(called_kw["client_info"], client_info) + self.assertEqual(called_kw["client_options"], client_options) self.assertIn("transport", called_kw) self.assertNotIn("credentials", called_kw) @@ -370,16 +377,17 @@ def test_database_admin_api_emulator_env(self, mock_em): called_args, called_kw = database_admin_client.call_args self.assertEqual(called_args, ()) self.assertEqual(called_kw["client_info"], client_info) + self.assertEqual(called_kw["client_options"], client_options) self.assertIn("transport", called_kw) - self.assertNotIn("client_options", called_kw) self.assertNotIn("credentials", called_kw) def test_database_admin_api_emulator_code(self): from google.auth.credentials import AnonymousCredentials + from google.api_core.client_options import ClientOptions credentials = AnonymousCredentials() client_info = mock.Mock() - client_options = {"api_endpoint": "emulator.host"} + client_options = ClientOptions(api_endpoint="emulator.host") client = self._make_one( project=self.PROJECT, credentials=credentials, @@ -401,8 +409,8 @@ def test_database_admin_api_emulator_code(self): called_args, called_kw = database_admin_client.call_args self.assertEqual(called_args, ()) self.assertEqual(called_kw["client_info"], client_info) + self.assertEqual(called_kw["client_options"], client_options) self.assertIn("transport", called_kw) - self.assertNotIn("client_options", called_kw) self.assertNotIn("credentials", called_kw) def test_copy(self):