From 4b63e7f2f458758edbaa2fd5b406cb47764d52f3 Mon Sep 17 00:00:00 2001 From: Weiran Fang Date: Wed, 27 Jun 2018 21:06:27 -0700 Subject: [PATCH 1/8] Add support for grpc_gcp secure channel --- api_core/google/api_core/grpc_helpers.py | 57 ++++++++++++++++++++++-- api_core/tests/unit/test_grpc_helpers.py | 50 +++++++++++++++------ 2 files changed, 89 insertions(+), 18 deletions(-) diff --git a/api_core/google/api_core/grpc_helpers.py b/api_core/google/api_core/grpc_helpers.py index 6a2f052f8403..282ef88de2ef 100644 --- a/api_core/google/api_core/grpc_helpers.py +++ b/api_core/google/api_core/grpc_helpers.py @@ -26,6 +26,11 @@ import google.auth.transport.grpc import google.auth.transport.requests +try: + import grpc_gcp + HAS_GRPC_GCP = True +except ImportError: + HAS_GRPC_GCP = False # The list of gRPC Callable interfaces that return iterators. _STREAM_WRAP_CLASSES = ( @@ -149,6 +154,51 @@ def wrap_errors(callable_): return _wrap_unary_errors(callable_) +def _create_secure_channel( + credentials, request, target, ssl_credentials=None, **kwargs): + """Creates a secure authorized gRPC channel. + + This overwrites google.auth.transport.grpc.secure_authorized_channel to + return a secure channel using grpc_gcp.secure_channel if grpc_gcp is + available. + + Args: + credentials (google.auth.credentials.Credentials): The credentials to + add to requests. + request (google.auth.transport.Request): A HTTP transport request + object used to refresh credentials as needed. Even though gRPC + is a separate transport, there's no way to refresh the credentials + without using a standard http transport. + target (str): The host and port of the service. + ssl_credentials (grpc.ChannelCredentials): Optional SSL channel + credentials. This can be used to specify different certificates. + kwargs: Additional arguments to pass to :func:`grpc_gcp.secure_channel`. + + Returns: + grpc_gcp.Channel: The created gRPC channel. + """ + # Create the metadata plugin for inserting the authorization header. + metadata_plugin = google.auth.transport.grpc.AuthMetadataPlugin( + credentials, request) + + # Create a set of grpc.CallCredentials using the metadata plugin. + google_auth_credentials = grpc.metadata_call_credentials(metadata_plugin) + + if ssl_credentials is None: + ssl_credentials = grpc.ssl_channel_credentials() + + # Combine the ssl credentials and the authorization credentials. + composite_credentials = grpc.composite_channel_credentials( + ssl_credentials, google_auth_credentials) + + if HAS_GRPC_GCP: + # If grpc_gcp module is available use grpc_gcp.secure_channel, + # otherwise, use grpc.secure_channel to create grpc channel. + return grpc_gcp.secure_channel(target, composite_credentials, **kwargs) + else: + return grpc.secure_channel(target, composite_credentials, **kwargs) + + def create_channel(target, credentials=None, scopes=None, **kwargs): """Create a secure channel with credentials. @@ -161,7 +211,7 @@ def create_channel(target, credentials=None, scopes=None, **kwargs): service. These are only used when credentials are not specified and are passed to :func:`google.auth.default`. kwargs: Additional key-word args passed to - :func:`google.auth.transport.grpc.secure_authorized_channel`. + :func:`_create_secure_channel`. Returns: grpc.Channel: The created channel. @@ -171,11 +221,10 @@ def create_channel(target, credentials=None, scopes=None, **kwargs): else: credentials = google.auth.credentials.with_scopes_if_required( credentials, scopes) - + request = google.auth.transport.requests.Request() - return google.auth.transport.grpc.secure_authorized_channel( - credentials, request, target, **kwargs) + return _create_secure_channel(credentials, request, target, **kwargs) _MethodCall = collections.namedtuple( diff --git a/api_core/tests/unit/test_grpc_helpers.py b/api_core/tests/unit/test_grpc_helpers.py index e5e4311ee732..a95f121d1f73 100644 --- a/api_core/tests/unit/test_grpc_helpers.py +++ b/api_core/tests/unit/test_grpc_helpers.py @@ -21,6 +21,11 @@ import google.auth.credentials from google.longrunning import operations_pb2 +try: + import grpc_gcp + HAS_GRPC_GCP = True +except ImportError: + HAS_GRPC_GCP = False def test__patch_callable_name(): callable = mock.Mock(spec=['__class__']) @@ -175,50 +180,49 @@ def test_wrap_errors_streaming(wrap_stream_errors): assert result == wrap_stream_errors.return_value wrap_stream_errors.assert_called_once_with(callable_) - @mock.patch( 'google.auth.default', return_value=(mock.sentinel.credentials, mock.sentinel.projet)) -@mock.patch('google.auth.transport.grpc.secure_authorized_channel') -def test_create_channel_implicit(secure_authorized_channel, default): +@mock.patch('google.api_core.grpc_helpers._create_secure_channel') +def test_create_channel_implicit(create_secure_channel, default): target = 'example.com:443' channel = grpc_helpers.create_channel(target) - assert channel is secure_authorized_channel.return_value + assert channel is create_secure_channel.return_value default.assert_called_once_with(scopes=None) - secure_authorized_channel.assert_called_once_with( + create_secure_channel.assert_called_once_with( mock.sentinel.credentials, mock.ANY, target) @mock.patch( 'google.auth.default', return_value=(mock.sentinel.credentials, mock.sentinel.projet)) -@mock.patch('google.auth.transport.grpc.secure_authorized_channel') +@mock.patch('google.api_core.grpc_helpers._create_secure_channel') def test_create_channel_implicit_with_scopes( - secure_authorized_channel, default): + create_secure_channel, default): target = 'example.com:443' channel = grpc_helpers.create_channel(target, scopes=['one', 'two']) - assert channel is secure_authorized_channel.return_value + assert channel is create_secure_channel.return_value default.assert_called_once_with(scopes=['one', 'two']) -@mock.patch('google.auth.transport.grpc.secure_authorized_channel') -def test_create_channel_explicit(secure_authorized_channel): +@mock.patch('google.api_core.grpc_helpers._create_secure_channel') +def test_create_channel_explicit(create_secure_channel): target = 'example.com:443' channel = grpc_helpers.create_channel( target, credentials=mock.sentinel.credentials) - assert channel is secure_authorized_channel.return_value - secure_authorized_channel.assert_called_once_with( + assert channel is create_secure_channel.return_value + create_secure_channel.assert_called_once_with( mock.sentinel.credentials, mock.ANY, target) -@mock.patch('google.auth.transport.grpc.secure_authorized_channel') -def test_create_channel_explicit_scoped(unused_secure_authorized_channel): +@mock.patch('google.api_core.grpc_helpers._create_secure_channel') +def test_create_channel_explicit_scoped(unused_create_secure_channel): scopes = ['1', '2'] credentials = mock.create_autospec( @@ -233,6 +237,24 @@ def test_create_channel_explicit_scoped(unused_secure_authorized_channel): credentials.with_scopes.assert_called_once_with(scopes) +@pytest.mark.skipif(not HAS_GRPC_GCP, reason='grpc_gcp module not available') +@mock.patch('grpc_gcp.secure_channel') +def test_create_channel_with_grpc_gcp(grpc_gcp_secure_channel): + target = 'example.com:443' + scopes = ['test_scope'] + + credentials = mock.create_autospec( + google.auth.credentials.Scoped, instance=True) + credentials.requires_scopes = True + + channel = grpc_helpers.create_channel( + target, + credentials=credentials, + scopes=scopes) + grpc_gcp_secure_channel.assert_called() + credentials.with_scopes.assert_called_once_with(scopes) + + class TestChannelStub(object): def test_single_response(self): From 2d776f0fc6aad4ab63ee4a8c8f8e49ebc682b103 Mon Sep 17 00:00:00 2001 From: Weiran Fang Date: Thu, 28 Jun 2018 15:45:07 -0700 Subject: [PATCH 2/8] Optional connection management support for spanner client --- spanner/MANIFEST.in | 2 +- .../spanner_v1/gapic/spanner.grpc.config | 88 +++++++++++++++++++ .../cloud/spanner_v1/gapic/spanner_client.py | 20 +++++ .../unit/gapic/v1/test_spanner_client_v1.py | 23 +++++ 4 files changed, 132 insertions(+), 1 deletion(-) create mode 100755 spanner/google/cloud/spanner_v1/gapic/spanner.grpc.config diff --git a/spanner/MANIFEST.in b/spanner/MANIFEST.in index fc77f8c82ff0..b6ebc267ccf6 100644 --- a/spanner/MANIFEST.in +++ b/spanner/MANIFEST.in @@ -1,4 +1,4 @@ include README.rst LICENSE -recursive-include google *.json *.proto +recursive-include google *.json *.proto *.config recursive-include tests * global-exclude *.pyc __pycache__ diff --git a/spanner/google/cloud/spanner_v1/gapic/spanner.grpc.config b/spanner/google/cloud/spanner_v1/gapic/spanner.grpc.config new file mode 100755 index 000000000000..c34397a1c869 --- /dev/null +++ b/spanner/google/cloud/spanner_v1/gapic/spanner.grpc.config @@ -0,0 +1,88 @@ +channel_pool: { + max_size: 10 + max_concurrent_streams_low_watermark: 100 +} +method: { + name: "/google.spanner.v1.Spanner/CreateSession" + affinity: { + command: BIND + affinity_key: "name" + } +} +method: { + name: "/google.spanner.v1.Spanner/GetSession" + affinity: { + command: BOUND + affinity_key: "name" + } +} +method: { + name: "/google.spanner.v1.Spanner/DeleteSession" + affinity: { + command: UNBIND + affinity_key: "name" + } +} +method: { + name: "/google.spanner.v1.Spanner/ExecuteSql" + affinity: { + command: BOUND + affinity_key: "session" + } +} +method: { + name: "/google.spanner.v1.Spanner/ExecuteStreamingSql" + affinity: { + command: BOUND + affinity_key: "session" + } +} +method: { + name: "/google.spanner.v1.Spanner/Read" + affinity: { + command: BOUND + affinity_key: "session" + } +} +method: { + name: "/google.spanner.v1.Spanner/StreamingRead" + affinity: { + command: BOUND + affinity_key: "session" + } +} +method: { + name: "/google.spanner.v1.Spanner/BeginTransaction" + affinity: { + command: BOUND + affinity_key: "session" + } +} +method: { + name: "/google.spanner.v1.Spanner/Commit" + affinity: { + command: BOUND + affinity_key: "session" + } +} +method: { + name: "/google.spanner.v1.Spanner/Rollback" + affinity: { + command: BOUND + affinity_key: "session" + } +} +method: { + name: "/google.spanner.v1.Spanner/PartitionQuery" + affinity: { + command: BOUND + affinity_key: "session" + } +} +method: { + name: "/google.spanner.v1.Spanner/PartitionRead" + affinity: { + command: BOUND + affinity_key: "session" + } +} diff --git a/spanner/google/cloud/spanner_v1/gapic/spanner_client.py b/spanner/google/cloud/spanner_v1/gapic/spanner_client.py index c88e97c7b11b..a037bb9fd32c 100644 --- a/spanner/google/cloud/spanner_v1/gapic/spanner_client.py +++ b/spanner/google/cloud/spanner_v1/gapic/spanner_client.py @@ -31,9 +31,17 @@ from google.cloud.spanner_v1.proto import spanner_pb2 from google.cloud.spanner_v1.proto import transaction_pb2 from google.protobuf import struct_pb2 +from google.protobuf import text_format + +try: + import grpc_gcp + HAS_GRPC_GCP = True +except ImportError: + HAS_GRPC_GCP = False _GAPIC_LIBRARY_VERSION = pkg_resources.get_distribution( 'google-cloud-spanner', ).version +_SPANNER_GRPC_CONFIG = 'spanner.grpc.config' class SpannerClient(object): @@ -113,10 +121,22 @@ def __init__(self, # Create the channel. if channel is None: + options = None + + if HAS_GRPC_GCP: + # Initialize grpc gcp config for spanner api. + grpc_gcp_config = grpc_gcp.proto.grpc_gcp_pb2.ApiConfig() + text_format.Merge( + pkg_resources.resource_string(__name__, _SPANNER_GRPC_CONFIG), + grpc_gcp_config + ) + options = [(grpc_gcp.API_CONFIG_CHANNEL_ARG, grpc_gcp_config)] + channel = google.api_core.grpc_helpers.create_channel( self.SERVICE_ADDRESS, credentials=credentials, scopes=self._DEFAULT_SCOPES, + options=options, ) # Create the gRPC stubs. diff --git a/spanner/tests/unit/gapic/v1/test_spanner_client_v1.py b/spanner/tests/unit/gapic/v1/test_spanner_client_v1.py index c770c3d1e8da..dc8842af18bc 100644 --- a/spanner/tests/unit/gapic/v1/test_spanner_client_v1.py +++ b/spanner/tests/unit/gapic/v1/test_spanner_client_v1.py @@ -13,6 +13,7 @@ # limitations under the License. """Unit tests.""" +import mock import pytest # Manual edit to auto-generated import because we do not expose the @@ -25,6 +26,11 @@ from google.cloud.spanner_v1.proto import transaction_pb2 from google.protobuf import empty_pb2 +try: + import grpc_gcp + HAS_GRPC_GCP = True +except ImportError: + HAS_GRPC_GCP = False class MultiCallableStub(object): """Stub for the grpc.UnaryUnaryMultiCallable interface.""" @@ -554,3 +560,20 @@ def test_partition_read_exception(self): with pytest.raises(CustomException): client.partition_read(session, table, key_set) + + @pytest.mark.skipif(not HAS_GRPC_GCP, + reason='grpc_gcp module not available') + @mock.patch('google.protobuf.text_format.Merge') + @mock.patch('grpc_gcp.proto.grpc_gcp_pb2.ApiConfig', + return_value=mock.sentinel.api_config) + @mock.patch('grpc_gcp.secure_channel') + def test_client_with_grpc_gcp_channel(self, + grpc_gcp_secure_channel, + api_config, + merge): + spanner_target = 'spanner.googleapis.com:443' + client = spanner_v1.SpannerClient() + merge.assert_called_once_with(mock.ANY, mock.sentinel.api_config) + options = [('grpc_gcp.api_config', mock.sentinel.api_config)] + grpc_gcp_secure_channel.assert_called_once_with( + spanner_target, mock.ANY, options=options) From 82c05feb4080d0191985bafbaeb6d2975ca2e384 Mon Sep 17 00:00:00 2001 From: Weiran Fang Date: Thu, 28 Jun 2018 22:05:14 -0700 Subject: [PATCH 3/8] Add nox tests and make sure 100% coverage. --- api_core/google/api_core/grpc_helpers.py | 9 ++-- api_core/nox.py | 11 +++++ api_core/tests/unit/test_grpc_helpers.py | 48 ++++++++++++++++--- spanner/nox.py | 9 ++++ .../unit/gapic/v1/test_spanner_client_v1.py | 7 +-- 5 files changed, 67 insertions(+), 17 deletions(-) diff --git a/api_core/google/api_core/grpc_helpers.py b/api_core/google/api_core/grpc_helpers.py index 282ef88de2ef..3f23a2e80b1d 100644 --- a/api_core/google/api_core/grpc_helpers.py +++ b/api_core/google/api_core/grpc_helpers.py @@ -157,7 +157,7 @@ def wrap_errors(callable_): def _create_secure_channel( credentials, request, target, ssl_credentials=None, **kwargs): """Creates a secure authorized gRPC channel. - + This overwrites google.auth.transport.grpc.secure_authorized_channel to return a secure channel using grpc_gcp.secure_channel if grpc_gcp is available. @@ -172,7 +172,8 @@ def _create_secure_channel( target (str): The host and port of the service. ssl_credentials (grpc.ChannelCredentials): Optional SSL channel credentials. This can be used to specify different certificates. - kwargs: Additional arguments to pass to :func:`grpc_gcp.secure_channel`. + kwargs: Additional arguments to pass to + :func:`grpc_gcp.secure_channel`. Returns: grpc_gcp.Channel: The created gRPC channel. @@ -190,7 +191,7 @@ def _create_secure_channel( # Combine the ssl credentials and the authorization credentials. composite_credentials = grpc.composite_channel_credentials( ssl_credentials, google_auth_credentials) - + if HAS_GRPC_GCP: # If grpc_gcp module is available use grpc_gcp.secure_channel, # otherwise, use grpc.secure_channel to create grpc channel. @@ -221,7 +222,7 @@ def create_channel(target, credentials=None, scopes=None, **kwargs): else: credentials = google.auth.credentials.with_scopes_if_required( credentials, scopes) - + request = google.auth.transport.requests.Request() return _create_secure_channel(credentials, request, target, **kwargs) diff --git a/api_core/nox.py b/api_core/nox.py index 7b40c6821023..6744efc46a65 100644 --- a/api_core/nox.py +++ b/api_core/nox.py @@ -65,6 +65,17 @@ def unit(session, py): default(session) +@nox.session +@nox.parametrize('py', ['2.7', '3.5', '3.6', '3.7']) +def unit_with_grpc_gcp(session, py): + """Run the unit test suite with grpcio-gcp installed.""" + + # Install grpcio-gcp + session.install('grpcio-gcp') + + unit(session, py) + + @nox.session def lint(session): """Run linters. diff --git a/api_core/tests/unit/test_grpc_helpers.py b/api_core/tests/unit/test_grpc_helpers.py index a95f121d1f73..45dc1f1e2013 100644 --- a/api_core/tests/unit/test_grpc_helpers.py +++ b/api_core/tests/unit/test_grpc_helpers.py @@ -21,11 +21,6 @@ import google.auth.credentials from google.longrunning import operations_pb2 -try: - import grpc_gcp - HAS_GRPC_GCP = True -except ImportError: - HAS_GRPC_GCP = False def test__patch_callable_name(): callable = mock.Mock(spec=['__class__']) @@ -180,6 +175,7 @@ def test_wrap_errors_streaming(wrap_stream_errors): assert result == wrap_stream_errors.return_value wrap_stream_errors.assert_called_once_with(callable_) + @mock.patch( 'google.auth.default', return_value=(mock.sentinel.credentials, mock.sentinel.projet)) @@ -195,6 +191,24 @@ def test_create_channel_implicit(create_secure_channel, default): mock.sentinel.credentials, mock.ANY, target) +@mock.patch('grpc._channel.Channel') +@mock.patch('grpc.composite_channel_credentials') +@mock.patch( + 'google.auth.default', + return_value=(mock.sentinel.credentials, mock.sentinel.projet)) +def test_create_channel_implicit_with_ssl_creds( + default, composite, grpc_channel): + target = 'example.com:443' + + ssl_creds = grpc.ssl_channel_credentials() + + grpc_helpers.create_channel(target, ssl_credentials=ssl_creds) + + default.assert_called_once_with(scopes=None) + composite.assert_called_once_with(ssl_creds, mock.ANY) + grpc_channel.assert_called() + + @mock.patch( 'google.auth.default', return_value=(mock.sentinel.credentials, mock.sentinel.projet)) @@ -237,7 +251,8 @@ def test_create_channel_explicit_scoped(unused_create_secure_channel): credentials.with_scopes.assert_called_once_with(scopes) -@pytest.mark.skipif(not HAS_GRPC_GCP, reason='grpc_gcp module not available') +@pytest.mark.skipif(not grpc_helpers.HAS_GRPC_GCP, + reason='grpc_gcp module not available') @mock.patch('grpc_gcp.secure_channel') def test_create_channel_with_grpc_gcp(grpc_gcp_secure_channel): target = 'example.com:443' @@ -247,7 +262,7 @@ def test_create_channel_with_grpc_gcp(grpc_gcp_secure_channel): google.auth.credentials.Scoped, instance=True) credentials.requires_scopes = True - channel = grpc_helpers.create_channel( + grpc_helpers.create_channel( target, credentials=credentials, scopes=scopes) @@ -255,6 +270,25 @@ def test_create_channel_with_grpc_gcp(grpc_gcp_secure_channel): credentials.with_scopes.assert_called_once_with(scopes) +@pytest.mark.skipif(grpc_helpers.HAS_GRPC_GCP, + reason='grpc_gcp module not available') +@mock.patch('grpc.secure_channel') +def test_create_channel_without_grpc_gcp(grpc_secure_channel): + target = 'example.com:443' + scopes = ['test_scope'] + + credentials = mock.create_autospec( + google.auth.credentials.Scoped, instance=True) + credentials.requires_scopes = True + + grpc_helpers.create_channel( + target, + credentials=credentials, + scopes=scopes) + grpc_secure_channel.assert_called() + credentials.with_scopes.assert_called_once_with(scopes) + + class TestChannelStub(object): def test_single_response(self): diff --git a/spanner/nox.py b/spanner/nox.py index 98f009c30b2c..bb0f48c95dac 100644 --- a/spanner/nox.py +++ b/spanner/nox.py @@ -67,6 +67,15 @@ def unit(session, py): default(session) +@nox.session +@nox.parametrize('py', ['2.7', '3.4', '3.5', '3.6', '3.7']) +def unit_with_grpc_gcp(session, py): + """Run the unit test suite with grpcio-gcp installed.""" + + # Install grpcio-gcp + session.install('grpcio-gcp') + + unit(session, py) @nox.session @nox.parametrize('py', ['2.7', '3.6']) diff --git a/spanner/tests/unit/gapic/v1/test_spanner_client_v1.py b/spanner/tests/unit/gapic/v1/test_spanner_client_v1.py index dc8842af18bc..876ecdb5c5b2 100644 --- a/spanner/tests/unit/gapic/v1/test_spanner_client_v1.py +++ b/spanner/tests/unit/gapic/v1/test_spanner_client_v1.py @@ -26,11 +26,6 @@ from google.cloud.spanner_v1.proto import transaction_pb2 from google.protobuf import empty_pb2 -try: - import grpc_gcp - HAS_GRPC_GCP = True -except ImportError: - HAS_GRPC_GCP = False class MultiCallableStub(object): """Stub for the grpc.UnaryUnaryMultiCallable interface.""" @@ -561,7 +556,7 @@ def test_partition_read_exception(self): with pytest.raises(CustomException): client.partition_read(session, table, key_set) - @pytest.mark.skipif(not HAS_GRPC_GCP, + @pytest.mark.skipif(not spanner_v1.HAS_GRPC_GCP, reason='grpc_gcp module not available') @mock.patch('google.protobuf.text_format.Merge') @mock.patch('grpc_gcp.proto.grpc_gcp_pb2.ApiConfig', From 66ce7157343b774999535b2082a0c8d78af51afe Mon Sep 17 00:00:00 2001 From: Weiran Fang Date: Fri, 29 Jun 2018 16:13:51 -0700 Subject: [PATCH 4/8] Remove _create_secure_channel function, use helper function to create channel config, add system tests with grpc_gcp --- .gitignore | 3 + api_core/google/api_core/grpc_helpers.py | 71 +++++++------------ api_core/nox.py | 10 ++- api_core/tests/unit/test_grpc_helpers.py | 55 +++++++++----- .../cloud/spanner_v1/gapic/spanner_client.py | 9 +-- spanner/nox.py | 55 +++++++++++--- .../unit/gapic/v1/test_spanner_client_v1.py | 6 +- 7 files changed, 125 insertions(+), 84 deletions(-) diff --git a/.gitignore b/.gitignore index 6fba7d3e252c..295771ac79ae 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,9 @@ htmlcov # JetBrains .idea +# VS Code +.vscode + # Built documentation docs/_build docs/_build_doc2dash diff --git a/api_core/google/api_core/grpc_helpers.py b/api_core/google/api_core/grpc_helpers.py index 3f23a2e80b1d..506e519abcc4 100644 --- a/api_core/google/api_core/grpc_helpers.py +++ b/api_core/google/api_core/grpc_helpers.py @@ -154,30 +154,35 @@ def wrap_errors(callable_): return _wrap_unary_errors(callable_) -def _create_secure_channel( - credentials, request, target, ssl_credentials=None, **kwargs): - """Creates a secure authorized gRPC channel. - - This overwrites google.auth.transport.grpc.secure_authorized_channel to - return a secure channel using grpc_gcp.secure_channel if grpc_gcp is - available. +def create_channel(target, + credentials=None, + scopes=None, + ssl_credentials=None, + **kwargs): + """Create a secure channel with credentials. Args: - credentials (google.auth.credentials.Credentials): The credentials to - add to requests. - request (google.auth.transport.Request): A HTTP transport request - object used to refresh credentials as needed. Even though gRPC - is a separate transport, there's no way to refresh the credentials - without using a standard http transport. - target (str): The host and port of the service. - ssl_credentials (grpc.ChannelCredentials): Optional SSL channel - credentials. This can be used to specify different certificates. - kwargs: Additional arguments to pass to - :func:`grpc_gcp.secure_channel`. + target (str): The target service address in the format 'hostname:port'. + credentials (google.auth.credentials.Credentials): The credentials. If + not specified, then this function will attempt to ascertain the + credentials from the environment using :func:`google.auth.default`. + scopes (Sequence[str]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + kwargs: Additional key-word args passed to + :func:`grpc_gcp.secure_channel` or :func:`grpc.secure_channel`. Returns: - grpc_gcp.Channel: The created gRPC channel. + grpc.Channel: The created channel. """ + if credentials is None: + credentials, _ = google.auth.default(scopes=scopes) + else: + credentials = google.auth.credentials.with_scopes_if_required( + credentials, scopes) + + request = google.auth.transport.requests.Request() + # Create the metadata plugin for inserting the authorization header. metadata_plugin = google.auth.transport.grpc.AuthMetadataPlugin( credentials, request) @@ -200,34 +205,6 @@ def _create_secure_channel( return grpc.secure_channel(target, composite_credentials, **kwargs) -def create_channel(target, credentials=None, scopes=None, **kwargs): - """Create a secure channel with credentials. - - Args: - target (str): The target service address in the format 'hostname:port'. - credentials (google.auth.credentials.Credentials): The credentials. If - not specified, then this function will attempt to ascertain the - credentials from the environment using :func:`google.auth.default`. - scopes (Sequence[str]): A optional list of scopes needed for this - service. These are only used when credentials are not specified and - are passed to :func:`google.auth.default`. - kwargs: Additional key-word args passed to - :func:`_create_secure_channel`. - - Returns: - grpc.Channel: The created channel. - """ - if credentials is None: - credentials, _ = google.auth.default(scopes=scopes) - else: - credentials = google.auth.credentials.with_scopes_if_required( - credentials, scopes) - - request = google.auth.transport.requests.Request() - - return _create_secure_channel(credentials, request, target, **kwargs) - - _MethodCall = collections.namedtuple( '_MethodCall', ('request', 'timeout', 'metadata', 'credentials')) diff --git a/api_core/nox.py b/api_core/nox.py index 6744efc46a65..8289c12fce9b 100644 --- a/api_core/nox.py +++ b/api_core/nox.py @@ -67,13 +67,19 @@ def unit(session, py): @nox.session @nox.parametrize('py', ['2.7', '3.5', '3.6', '3.7']) -def unit_with_grpc_gcp(session, py): +def unit_grpc_gcp(session, py): """Run the unit test suite with grpcio-gcp installed.""" + # Run unit tests against all supported versions of Python. + session.interpreter = 'python{}'.format(py) + + # Set the virtualenv dirname. + session.virtualenv_dirname = 'unit-grpc-gcp-' + py + # Install grpcio-gcp session.install('grpcio-gcp') - unit(session, py) + default(session, py) @nox.session diff --git a/api_core/tests/unit/test_grpc_helpers.py b/api_core/tests/unit/test_grpc_helpers.py index 45dc1f1e2013..f81c3000968b 100644 --- a/api_core/tests/unit/test_grpc_helpers.py +++ b/api_core/tests/unit/test_grpc_helpers.py @@ -176,19 +176,22 @@ def test_wrap_errors_streaming(wrap_stream_errors): wrap_stream_errors.assert_called_once_with(callable_) +@mock.patch('grpc.composite_channel_credentials') @mock.patch( 'google.auth.default', return_value=(mock.sentinel.credentials, mock.sentinel.projet)) -@mock.patch('google.api_core.grpc_helpers._create_secure_channel') -def test_create_channel_implicit(create_secure_channel, default): +@mock.patch('grpc._channel.Channel') +def test_create_channel_implicit(grpc_channel, default, composite_creds_call): target = 'example.com:443' + composite_creds = composite_creds_call.return_value + composite_creds._credentials = mock.sentinel.channel_creds channel = grpc_helpers.create_channel(target) - assert channel is create_secure_channel.return_value + assert channel is grpc_channel.return_value default.assert_called_once_with(scopes=None) - create_secure_channel.assert_called_once_with( - mock.sentinel.credentials, mock.ANY, target) + grpc_channel.assert_called_once_with( + target, (), mock.sentinel.channel_creds) @mock.patch('grpc._channel.Channel') @@ -209,46 +212,64 @@ def test_create_channel_implicit_with_ssl_creds( grpc_channel.assert_called() +@mock.patch('grpc.composite_channel_credentials') @mock.patch( 'google.auth.default', return_value=(mock.sentinel.credentials, mock.sentinel.projet)) -@mock.patch('google.api_core.grpc_helpers._create_secure_channel') +@mock.patch('grpc._channel.Channel') def test_create_channel_implicit_with_scopes( - create_secure_channel, default): + grpc_channel, default, composite_creds_call): target = 'example.com:443' + composite_creds = composite_creds_call.return_value + composite_creds._credentials = mock.sentinel.channel_creds channel = grpc_helpers.create_channel(target, scopes=['one', 'two']) - assert channel is create_secure_channel.return_value + assert channel is grpc_channel.return_value default.assert_called_once_with(scopes=['one', 'two']) + grpc_channel.assert_called_once_with( + target, (), mock.sentinel.channel_creds) -@mock.patch('google.api_core.grpc_helpers._create_secure_channel') -def test_create_channel_explicit(create_secure_channel): +@mock.patch('grpc.composite_channel_credentials') +@mock.patch('google.auth.credentials.with_scopes_if_required') +@mock.patch('grpc._channel.Channel') +def test_create_channel_explicit( + grpc_channel, auth_creds, composite_creds_call): target = 'example.com:443' + composite_creds = composite_creds_call.return_value + composite_creds._credentials = mock.sentinel.channel_creds channel = grpc_helpers.create_channel( target, credentials=mock.sentinel.credentials) - assert channel is create_secure_channel.return_value - create_secure_channel.assert_called_once_with( - mock.sentinel.credentials, mock.ANY, target) + auth_creds.assert_called_once_with(mock.sentinel.credentials, None) + assert channel is grpc_channel.return_value + grpc_channel.assert_called_once_with( + target, (), mock.sentinel.channel_creds) -@mock.patch('google.api_core.grpc_helpers._create_secure_channel') -def test_create_channel_explicit_scoped(unused_create_secure_channel): +@mock.patch('grpc.composite_channel_credentials') +@mock.patch('grpc._channel.Channel') +def test_create_channel_explicit_scoped(grpc_channel, composite_creds_call): + target = 'example.com:443' scopes = ['1', '2'] + composite_creds = composite_creds_call.return_value + composite_creds._credentials = mock.sentinel.channel_creds credentials = mock.create_autospec( google.auth.credentials.Scoped, instance=True) credentials.requires_scopes = True - grpc_helpers.create_channel( - mock.sentinel.target, + channel = grpc_helpers.create_channel( + target, credentials=credentials, scopes=scopes) credentials.with_scopes.assert_called_once_with(scopes) + assert channel is grpc_channel.return_value + grpc_channel.assert_called_once_with( + target, (), mock.sentinel.channel_creds) @pytest.mark.skipif(not grpc_helpers.HAS_GRPC_GCP, diff --git a/spanner/google/cloud/spanner_v1/gapic/spanner_client.py b/spanner/google/cloud/spanner_v1/gapic/spanner_client.py index a037bb9fd32c..cc4734d2b209 100644 --- a/spanner/google/cloud/spanner_v1/gapic/spanner_client.py +++ b/spanner/google/cloud/spanner_v1/gapic/spanner_client.py @@ -31,7 +31,6 @@ from google.cloud.spanner_v1.proto import spanner_pb2 from google.cloud.spanner_v1.proto import transaction_pb2 from google.protobuf import struct_pb2 -from google.protobuf import text_format try: import grpc_gcp @@ -125,11 +124,9 @@ def __init__(self, if HAS_GRPC_GCP: # Initialize grpc gcp config for spanner api. - grpc_gcp_config = grpc_gcp.proto.grpc_gcp_pb2.ApiConfig() - text_format.Merge( - pkg_resources.resource_string(__name__, _SPANNER_GRPC_CONFIG), - grpc_gcp_config - ) + grpc_gcp_config = grpc_gcp.api_config_from_text_pb( + pkg_resources.resource_string(__name__, + _SPANNER_GRPC_CONFIG)) options = [(grpc_gcp.API_CONFIG_CHANNEL_ARG, grpc_gcp_config)] channel = google.api_core.grpc_helpers.create_channel( diff --git a/spanner/nox.py b/spanner/nox.py index bb0f48c95dac..a1be1fd0c908 100644 --- a/spanner/nox.py +++ b/spanner/nox.py @@ -67,15 +67,37 @@ def unit(session, py): default(session) + @nox.session @nox.parametrize('py', ['2.7', '3.4', '3.5', '3.6', '3.7']) -def unit_with_grpc_gcp(session, py): +def unit_grpc_gcp(session, py): """Run the unit test suite with grpcio-gcp installed.""" + # Run unit tests against all supported versions of Python. + session.interpreter = 'python{}'.format(py) + + # Set the virtualenv dirname. + session.virtualenv_dirname = 'unit-grpc-gcp-' + py + # Install grpcio-gcp session.install('grpcio-gcp') - unit(session, py) + default(session) + + +def system_common(session): + # Use pre-release gRPC for system tests. + session.install('--pre', 'grpcio') + + # Install all test dependencies, then install this package into the + # virtualenv's dist-packages. + session.install('mock', 'pytest', *LOCAL_DEPS) + session.install('../test_utils/') + session.install('-e', '.') + + # Run py.test against the system tests. + session.run('py.test', '--quiet', 'tests/system', *session.posargs) + @nox.session @nox.parametrize('py', ['2.7', '3.6']) @@ -92,17 +114,28 @@ def system(session, py): # Set the virtualenv dirname. session.virtualenv_dirname = 'sys-' + py - # Use pre-release gRPC for system tests. - session.install('--pre', 'grpcio') + system_common(session) - # Install all test dependencies, then install this package into the - # virtualenv's dist-packages. - session.install('mock', 'pytest', *LOCAL_DEPS) - session.install('../test_utils/') - session.install('-e', '.') - # Run py.test against the system tests. - session.run('py.test', '--quiet', 'tests/system', *session.posargs) +@nox.session +@nox.parametrize('py', ['2.7', '3.6']) +def system_grpc_gcp(session, py): + """Run the system test suite with grpcio-gcp installed.""" + + # Sanity check: Only run system tests if the environment variable is set. + if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''): + session.skip('Credentials must be set via environment variable.') + + # Run the system tests against latest Python 2 and Python 3 only. + session.interpreter = 'python{}'.format(py) + + # Set the virtualenv dirname. + session.virtualenv_dirname = 'sys-grpc-gcp-' + py + + # Install grpcio-gcp + session.install('grpcio-gcp') + + system_common(session) @nox.session diff --git a/spanner/tests/unit/gapic/v1/test_spanner_client_v1.py b/spanner/tests/unit/gapic/v1/test_spanner_client_v1.py index 876ecdb5c5b2..bf082de883a6 100644 --- a/spanner/tests/unit/gapic/v1/test_spanner_client_v1.py +++ b/spanner/tests/unit/gapic/v1/test_spanner_client_v1.py @@ -558,6 +558,9 @@ def test_partition_read_exception(self): @pytest.mark.skipif(not spanner_v1.HAS_GRPC_GCP, reason='grpc_gcp module not available') + @mock.patch( + 'google.auth.default', + return_value=(mock.sentinel.credentials, mock.sentinel.projet)) @mock.patch('google.protobuf.text_format.Merge') @mock.patch('grpc_gcp.proto.grpc_gcp_pb2.ApiConfig', return_value=mock.sentinel.api_config) @@ -565,7 +568,8 @@ def test_partition_read_exception(self): def test_client_with_grpc_gcp_channel(self, grpc_gcp_secure_channel, api_config, - merge): + merge, + auth_default): spanner_target = 'spanner.googleapis.com:443' client = spanner_v1.SpannerClient() merge.assert_called_once_with(mock.ANY, mock.sentinel.api_config) From 5384c9199506a248d1e13a1e06c204a78f5abfad Mon Sep 17 00:00:00 2001 From: Weiran Fang Date: Fri, 29 Jun 2018 16:21:28 -0700 Subject: [PATCH 5/8] Fix api_core nox. --- api_core/nox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_core/nox.py b/api_core/nox.py index 8289c12fce9b..dacfbb53e74d 100644 --- a/api_core/nox.py +++ b/api_core/nox.py @@ -79,7 +79,7 @@ def unit_grpc_gcp(session, py): # Install grpcio-gcp session.install('grpcio-gcp') - default(session, py) + default(session) @nox.session From 3027eb3c3c0677861f9e9d173c0d97886747a957 Mon Sep 17 00:00:00 2001 From: Weiran Fang Date: Fri, 13 Jul 2018 13:56:28 -0700 Subject: [PATCH 6/8] Update grpc_helpers.create_channel docstring. --- api_core/google/api_core/grpc_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api_core/google/api_core/grpc_helpers.py b/api_core/google/api_core/grpc_helpers.py index 506e519abcc4..b4ac9e0750ec 100644 --- a/api_core/google/api_core/grpc_helpers.py +++ b/api_core/google/api_core/grpc_helpers.py @@ -169,6 +169,8 @@ def create_channel(target, scopes (Sequence[str]): A optional list of scopes needed for this service. These are only used when credentials are not specified and are passed to :func:`google.auth.default`. + ssl_credentials (grpc.ChannelCredentials): Optional SSL channel + credentials. This can be used to specify different certificates. kwargs: Additional key-word args passed to :func:`grpc_gcp.secure_channel` or :func:`grpc.secure_channel`. From ab6a9096419b25ae2b5d5086d3fa920e5b0fc59a Mon Sep 17 00:00:00 2001 From: Weiran Fang Date: Tue, 24 Jul 2018 13:48:32 -0700 Subject: [PATCH 7/8] Remove mocked private members --- api_core/tests/unit/test_grpc_helpers.py | 56 ++++++++++++------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/api_core/tests/unit/test_grpc_helpers.py b/api_core/tests/unit/test_grpc_helpers.py index f81c3000968b..d58a85cc8f00 100644 --- a/api_core/tests/unit/test_grpc_helpers.py +++ b/api_core/tests/unit/test_grpc_helpers.py @@ -180,27 +180,27 @@ def test_wrap_errors_streaming(wrap_stream_errors): @mock.patch( 'google.auth.default', return_value=(mock.sentinel.credentials, mock.sentinel.projet)) -@mock.patch('grpc._channel.Channel') -def test_create_channel_implicit(grpc_channel, default, composite_creds_call): +@mock.patch('grpc.secure_channel') +def test_create_channel_implicit( + grpc_secure_channel, default, composite_creds_call): target = 'example.com:443' composite_creds = composite_creds_call.return_value - composite_creds._credentials = mock.sentinel.channel_creds channel = grpc_helpers.create_channel(target) - assert channel is grpc_channel.return_value + assert channel is grpc_secure_channel.return_value default.assert_called_once_with(scopes=None) - grpc_channel.assert_called_once_with( - target, (), mock.sentinel.channel_creds) + grpc_secure_channel.assert_called_once_with( + target, composite_creds) -@mock.patch('grpc._channel.Channel') @mock.patch('grpc.composite_channel_credentials') @mock.patch( 'google.auth.default', return_value=(mock.sentinel.credentials, mock.sentinel.projet)) +@mock.patch('grpc.secure_channel') def test_create_channel_implicit_with_ssl_creds( - default, composite, grpc_channel): + grpc_secure_channel, default, composite_creds_call): target = 'example.com:443' ssl_creds = grpc.ssl_channel_credentials() @@ -208,54 +208,54 @@ def test_create_channel_implicit_with_ssl_creds( grpc_helpers.create_channel(target, ssl_credentials=ssl_creds) default.assert_called_once_with(scopes=None) - composite.assert_called_once_with(ssl_creds, mock.ANY) - grpc_channel.assert_called() + composite_creds_call.assert_called_once_with(ssl_creds, mock.ANY) + composite_creds = composite_creds_call.return_value + grpc_secure_channel.assert_called_once_with( + target, composite_creds) @mock.patch('grpc.composite_channel_credentials') @mock.patch( 'google.auth.default', return_value=(mock.sentinel.credentials, mock.sentinel.projet)) -@mock.patch('grpc._channel.Channel') +@mock.patch('grpc.secure_channel') def test_create_channel_implicit_with_scopes( - grpc_channel, default, composite_creds_call): + grpc_secure_channel, default, composite_creds_call): target = 'example.com:443' composite_creds = composite_creds_call.return_value - composite_creds._credentials = mock.sentinel.channel_creds channel = grpc_helpers.create_channel(target, scopes=['one', 'two']) - assert channel is grpc_channel.return_value + assert channel is grpc_secure_channel.return_value default.assert_called_once_with(scopes=['one', 'two']) - grpc_channel.assert_called_once_with( - target, (), mock.sentinel.channel_creds) + grpc_secure_channel.assert_called_once_with( + target, composite_creds) @mock.patch('grpc.composite_channel_credentials') @mock.patch('google.auth.credentials.with_scopes_if_required') -@mock.patch('grpc._channel.Channel') +@mock.patch('grpc.secure_channel') def test_create_channel_explicit( - grpc_channel, auth_creds, composite_creds_call): + grpc_secure_channel, auth_creds, composite_creds_call): target = 'example.com:443' composite_creds = composite_creds_call.return_value - composite_creds._credentials = mock.sentinel.channel_creds channel = grpc_helpers.create_channel( target, credentials=mock.sentinel.credentials) auth_creds.assert_called_once_with(mock.sentinel.credentials, None) - assert channel is grpc_channel.return_value - grpc_channel.assert_called_once_with( - target, (), mock.sentinel.channel_creds) + assert channel is grpc_secure_channel.return_value + grpc_secure_channel.assert_called_once_with( + target, composite_creds) @mock.patch('grpc.composite_channel_credentials') -@mock.patch('grpc._channel.Channel') -def test_create_channel_explicit_scoped(grpc_channel, composite_creds_call): +@mock.patch('grpc.secure_channel') +def test_create_channel_explicit_scoped( + grpc_secure_channel, composite_creds_call): target = 'example.com:443' scopes = ['1', '2'] composite_creds = composite_creds_call.return_value - composite_creds._credentials = mock.sentinel.channel_creds credentials = mock.create_autospec( google.auth.credentials.Scoped, instance=True) @@ -267,9 +267,9 @@ def test_create_channel_explicit_scoped(grpc_channel, composite_creds_call): scopes=scopes) credentials.with_scopes.assert_called_once_with(scopes) - assert channel is grpc_channel.return_value - grpc_channel.assert_called_once_with( - target, (), mock.sentinel.channel_creds) + assert channel is grpc_secure_channel.return_value + grpc_secure_channel.assert_called_once_with( + target, composite_creds) @pytest.mark.skipif(not grpc_helpers.HAS_GRPC_GCP, From 0df45cb7fe9ce0a38766b981d78ea6fd1b5f22c8 Mon Sep 17 00:00:00 2001 From: Weiran Fang Date: Tue, 24 Jul 2018 14:12:20 -0700 Subject: [PATCH 8/8] Fix unit tests when grpc_gcp enabled --- api_core/tests/unit/test_grpc_helpers.py | 40 ++++++++++++++++++------ 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/api_core/tests/unit/test_grpc_helpers.py b/api_core/tests/unit/test_grpc_helpers.py index d58a85cc8f00..b91847c38646 100644 --- a/api_core/tests/unit/test_grpc_helpers.py +++ b/api_core/tests/unit/test_grpc_helpers.py @@ -190,8 +190,12 @@ def test_create_channel_implicit( assert channel is grpc_secure_channel.return_value default.assert_called_once_with(scopes=None) - grpc_secure_channel.assert_called_once_with( - target, composite_creds) + if (grpc_helpers.HAS_GRPC_GCP): + grpc_secure_channel.assert_called_once_with( + target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with( + target, composite_creds) @mock.patch('grpc.composite_channel_credentials') @@ -210,8 +214,12 @@ def test_create_channel_implicit_with_ssl_creds( default.assert_called_once_with(scopes=None) composite_creds_call.assert_called_once_with(ssl_creds, mock.ANY) composite_creds = composite_creds_call.return_value - grpc_secure_channel.assert_called_once_with( - target, composite_creds) + if (grpc_helpers.HAS_GRPC_GCP): + grpc_secure_channel.assert_called_once_with( + target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with( + target, composite_creds) @mock.patch('grpc.composite_channel_credentials') @@ -228,8 +236,12 @@ def test_create_channel_implicit_with_scopes( assert channel is grpc_secure_channel.return_value default.assert_called_once_with(scopes=['one', 'two']) - grpc_secure_channel.assert_called_once_with( - target, composite_creds) + if (grpc_helpers.HAS_GRPC_GCP): + grpc_secure_channel.assert_called_once_with( + target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with( + target, composite_creds) @mock.patch('grpc.composite_channel_credentials') @@ -245,8 +257,12 @@ def test_create_channel_explicit( auth_creds.assert_called_once_with(mock.sentinel.credentials, None) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with( - target, composite_creds) + if (grpc_helpers.HAS_GRPC_GCP): + grpc_secure_channel.assert_called_once_with( + target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with( + target, composite_creds) @mock.patch('grpc.composite_channel_credentials') @@ -268,8 +284,12 @@ def test_create_channel_explicit_scoped( credentials.with_scopes.assert_called_once_with(scopes) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with( - target, composite_creds) + if (grpc_helpers.HAS_GRPC_GCP): + grpc_secure_channel.assert_called_once_with( + target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with( + target, composite_creds) @pytest.mark.skipif(not grpc_helpers.HAS_GRPC_GCP,