From c45f4fc20f0fda260ba2ce743aeffc851f554bfa Mon Sep 17 00:00:00 2001 From: HemangChothani <50404902+HemangChothani@users.noreply.github.com> Date: Thu, 16 Apr 2020 03:13:57 +0530 Subject: [PATCH 01/16] docs(spanner): changes to support sphinx new release (#54) * docs(spanner): changes to support sphinx new release * docs(spanner): variable fix in changelog file * docs(spanner): nit --- CHANGELOG.md | 6 +++--- docs/conf.py | 1 + docs/gapic/v1/admin_database_types.rst | 1 + docs/gapic/v1/admin_instance_types.rst | 1 + docs/gapic/v1/types.rst | 1 + noxfile.py | 2 +- 6 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 713fe28347..3bcc901ac6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,12 +106,12 @@ Return sessions from pool in LIFO order. ([#9454](https://github.com/googleapis/ ### Implementation Changes -- Add backoff for `run_in_transaction' when backend does not provide 'RetryInfo' in response. ([#8461](https://github.com/googleapis/google-cloud-python/pull/8461)) +- Add backoff for `run_in_transaction` when backend does not provide 'RetryInfo' in response. ([#8461](https://github.com/googleapis/google-cloud-python/pull/8461)) - Adjust gRPC timeouts (via synth). ([#8445](https://github.com/googleapis/google-cloud-python/pull/8445)) - Allow kwargs to be passed to create_channel (via synth). ([#8403](https://github.com/googleapis/google-cloud-python/pull/8403)) ### New Features -- Add 'options_' argument to clients' 'get_iam_policy'; pin black version (via synth). ([#8659](https://github.com/googleapis/google-cloud-python/pull/8659)) +- Add 'options_' argument to clients 'get_iam_policy'; pin black version (via synth). ([#8659](https://github.com/googleapis/google-cloud-python/pull/8659)) - Add 'client_options' support, update list method docstrings (via synth). ([#8522](https://github.com/googleapis/google-cloud-python/pull/8522)) ### Dependencies @@ -382,6 +382,6 @@ Return sessions from pool in LIFO order. ([#9454](https://github.com/googleapis/ - Upgrading to `google-cloud-core >= 0.28.0` and adding dependency on `google-api-core` (#4221, #4280) - Deferring to `google-api-core` for `grpcio` and - `googleapis-common-protos`dependencies (#4096, #4098) + `googleapis-common-protos` dependencies (#4096, #4098) PyPI: https://pypi.org/project/google-cloud-spanner/0.29.0/ diff --git a/docs/conf.py b/docs/conf.py index e326daef4e..4fffc063c8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,6 +38,7 @@ "sphinx.ext.napoleon", "sphinx.ext.todo", "sphinx.ext.viewcode", + "recommonmark", ] # autodoc/autosummary flags diff --git a/docs/gapic/v1/admin_database_types.rst b/docs/gapic/v1/admin_database_types.rst index de3d9585c7..fa9aaa73b1 100644 --- a/docs/gapic/v1/admin_database_types.rst +++ b/docs/gapic/v1/admin_database_types.rst @@ -3,3 +3,4 @@ Spanner Admin Database Client Types .. automodule:: google.cloud.spanner_admin_database_v1.types :members: + :noindex: diff --git a/docs/gapic/v1/admin_instance_types.rst b/docs/gapic/v1/admin_instance_types.rst index 4cd06b3ca0..f8f3afa5ff 100644 --- a/docs/gapic/v1/admin_instance_types.rst +++ b/docs/gapic/v1/admin_instance_types.rst @@ -3,3 +3,4 @@ Spanner Admin Instance Client Types .. automodule:: google.cloud.spanner_admin_instance_v1.types :members: + :noindex: diff --git a/docs/gapic/v1/types.rst b/docs/gapic/v1/types.rst index 28956e60c7..54424febf3 100644 --- a/docs/gapic/v1/types.rst +++ b/docs/gapic/v1/types.rst @@ -3,3 +3,4 @@ Spanner Client Types .. automodule:: google.cloud.spanner_v1.types :members: + :noindex: diff --git a/noxfile.py b/noxfile.py index 88beb02d68..22f328c4af 100644 --- a/noxfile.py +++ b/noxfile.py @@ -143,7 +143,7 @@ def docs(session): """Build the docs for this library.""" session.install("-e", ".") - session.install("sphinx==2.4.4", "alabaster", "recommonmark") + session.install("sphinx", "alabaster", "recommonmark") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( From 805bbb766fd9c019f528e2f8ed1379d997622d03 Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Thu, 16 Apr 2020 13:24:00 +1200 Subject: [PATCH 02/16] fix: add keepalive changes to synth.py (#55) Co-authored-by: larkee --- synth.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/synth.py b/synth.py index c509089401..a351bbf943 100644 --- a/synth.py +++ b/synth.py @@ -46,7 +46,14 @@ s.replace( "google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py", "from google.cloud.spanner_v1.proto import spanner_pb2_grpc\n", - "\g<0>\n\n_SPANNER_GRPC_CONFIG = 'spanner.grpc.config'\n", + "\g<0>\n\n_GRPC_KEEPALIVE_MS = 2 * 60 * 1000\n" + "_SPANNER_GRPC_CONFIG = 'spanner.grpc.config'\n", +) + +s.replace( + "google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py", + "(\s+)'grpc.max_receive_message_length': -1,", + "\g<0>\g<1>\"grpc.keepalive_time_ms\": _GRPC_KEEPALIVE_MS,", ) s.replace( From 0f526bf8fb2a0ea258697110690a5a82b61c991c Mon Sep 17 00:00:00 2001 From: HemangChothani <50404902+HemangChothani@users.noreply.github.com> Date: Fri, 17 Apr 2020 03:04:14 +0530 Subject: [PATCH 03/16] docs(spanner): fix single character parameter (#48) * docs(spanner): fix single character parameter * docs(spanner): fix single character parameters * docs(spanner): fix lint * docs(spanner): nits * docs(spanner): nit --- google/cloud/spanner_v1/client.py | 7 ++++--- google/cloud/spanner_v1/database.py | 8 ++++---- google/cloud/spanner_v1/instance.py | 4 ++-- google/cloud/spanner_v1/session.py | 10 +++++----- google/cloud/spanner_v1/snapshot.py | 9 +++++---- google/cloud/spanner_v1/streamed.py | 8 ++++---- google/cloud/spanner_v1/transaction.py | 9 +++++---- 7 files changed, 29 insertions(+), 26 deletions(-) diff --git a/google/cloud/spanner_v1/client.py b/google/cloud/spanner_v1/client.py index 01b3ddfabf..29f3fcf69a 100644 --- a/google/cloud/spanner_v1/client.py +++ b/google/cloud/spanner_v1/client.py @@ -127,7 +127,7 @@ class Client(ClientWithProject): If none are specified, the client will attempt to ascertain the credentials from the environment. - :type client_info: :class:`google.api_core.gapic_v1.client_info.ClientInfo` + :type client_info: :class:`~google.api_core.gapic_v1.client_info.ClientInfo` :param client_info: (Optional) The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. Generally, @@ -145,7 +145,7 @@ class Client(ClientWithProject): on the client. API Endpoint should be set through client_options. :type query_options: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Query optimizer configuration to use for the given query. @@ -341,7 +341,8 @@ def instance( :param configuration_name: (Optional) Name of the instance configuration used to set up the instance's cluster, in the form: - ``projects//instanceConfigs/``. + ``projects//instanceConfigs/`` + ````. **Required** for instances which do not yet exist. :type display_name: str diff --git a/google/cloud/spanner_v1/database.py b/google/cloud/spanner_v1/database.py index 5785953bd7..99b7244f9d 100644 --- a/google/cloud/spanner_v1/database.py +++ b/google/cloud/spanner_v1/database.py @@ -129,7 +129,7 @@ def from_pb(cls, database_pb, instance, pool=None): """Creates an instance of this class from a protobuf. :type database_pb: - :class:`google.spanner.v2.spanner_instance_admin_pb2.Instance` + :class:`~google.spanner.v2.spanner_instance_admin_pb2.Instance` :param database_pb: A instance protobuf object. :type instance: :class:`~google.cloud.spanner_v1.instance.Instance` @@ -410,7 +410,7 @@ def execute_partitioned_dml( required if parameters are passed. :type query_options: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Query optimizer configuration to use for the given query. @@ -566,7 +566,7 @@ def restore(self, source): :type backup: :class:`~google.cloud.spanner_v1.backup.Backup` :param backup: the path of the backup being restored from. - :rtype: :class:'~google.api_core.operation.Operation` + :rtype: :class:`~google.api_core.operation.Operation` :returns: a future used to poll the status of the create request :raises Conflict: if the database already exists :raises NotFound: @@ -908,7 +908,7 @@ def generate_query_batches( differ. :type query_options: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Query optimizer configuration to use for the given query. diff --git a/google/cloud/spanner_v1/instance.py b/google/cloud/spanner_v1/instance.py index 4a14032c13..f0809e7d81 100644 --- a/google/cloud/spanner_v1/instance.py +++ b/google/cloud/spanner_v1/instance.py @@ -135,7 +135,7 @@ def from_pb(cls, instance_pb, client): """Creates an instance from a protobuf. :type instance_pb: - :class:`google.spanner.v2.spanner_instance_admin_pb2.Instance` + :class:`~google.spanner.v2.spanner_instance_admin_pb2.Instance` :param instance_pb: A instance protobuf object. :type client: :class:`~google.cloud.spanner_v1.client.Client` @@ -234,7 +234,7 @@ def create(self): before calling :meth:`create`. - :rtype: :class:`google.api_core.operation.Operation` + :rtype: :class:`~google.api_core.operation.Operation` :returns: an operation instance :raises Conflict: if the instance already exists """ diff --git a/google/cloud/spanner_v1/session.py b/google/cloud/spanner_v1/session.py index fc6bb028b7..61e4322012 100644 --- a/google/cloud/spanner_v1/session.py +++ b/google/cloud/spanner_v1/session.py @@ -216,18 +216,18 @@ def execute_sql( the names used in ``sql``. :type param_types: - dict, {str -> :class:`google.spanner.v1.type_pb2.TypeCode`} + dict, {str -> :class:`~google.spanner.v1.type_pb2.TypeCode`} :param param_types: (Optional) explicit types for one or more param values; overrides default type detection on the back-end. :type query_mode: - :class:`google.spanner.v1.spanner_pb2.ExecuteSqlRequest.QueryMode` - :param query_mode: Mode governing return of results / query plan. See - https://cloud.google.com/spanner/reference/rpc/google.spanner.v1#google.spanner.v1.ExecuteSqlRequest.QueryMode1 + :class:`~google.spanner.v1.spanner_pb2.ExecuteSqlRequest.QueryMode` + :param query_mode: Mode governing return of results / query plan. See: + `QueryMode `_. :type query_options: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Options that are provided for query plan stability. diff --git a/google/cloud/spanner_v1/snapshot.py b/google/cloud/spanner_v1/snapshot.py index 56b3b6a813..f7b9f07f8f 100644 --- a/google/cloud/spanner_v1/snapshot.py +++ b/google/cloud/spanner_v1/snapshot.py @@ -178,12 +178,13 @@ def execute_sql( required if parameters are passed. :type query_mode: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryMode` - :param query_mode: Mode governing return of results / query plan. See - https://cloud.google.com/spanner/reference/rpc/google.spanner.v1#google.spanner.v1.ExecuteSqlRequest.QueryMode1 + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryMode` + :param query_mode: Mode governing return of results / query plan. + See: + `QueryMode `_. :type query_options: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Query optimizer configuration to use for the given query. diff --git a/google/cloud/spanner_v1/streamed.py b/google/cloud/spanner_v1/streamed.py index 5d1a31e931..dbb4e0dbc0 100644 --- a/google/cloud/spanner_v1/streamed.py +++ b/google/cloud/spanner_v1/streamed.py @@ -32,7 +32,7 @@ class StreamedResultSet(object): :type response_iterator: :param response_iterator: Iterator yielding - :class:`google.cloud.spanner_v1.proto.result_set_pb2.PartialResultSet` + :class:`~google.cloud.spanner_v1.proto.result_set_pb2.PartialResultSet` instances. :type source: :class:`~google.cloud.spanner_v1.snapshot.Snapshot` @@ -195,13 +195,13 @@ def one_or_none(self): class Unmergeable(ValueError): """Unable to merge two values. - :type lhs: :class:`google.protobuf.struct_pb2.Value` + :type lhs: :class:`~google.protobuf.struct_pb2.Value` :param lhs: pending value to be merged - :type rhs: :class:`google.protobuf.struct_pb2.Value` + :type rhs: :class:`~google.protobuf.struct_pb2.Value` :param rhs: remaining value to be merged - :type type_: :class:`google.cloud.spanner_v1.proto.type_pb2.Type` + :type type_: :class:`~google.cloud.spanner_v1.proto.type_pb2.Type` :param type_: field type of values being merged """ diff --git a/google/cloud/spanner_v1/transaction.py b/google/cloud/spanner_v1/transaction.py index 27c260212e..3c1abc7326 100644 --- a/google/cloud/spanner_v1/transaction.py +++ b/google/cloud/spanner_v1/transaction.py @@ -183,12 +183,13 @@ def execute_update( required if parameters are passed. :type query_mode: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryMode` - :param query_mode: Mode governing return of results / query plan. See - https://cloud.google.com/spanner/reference/rpc/google.spanner.v1#google.spanner.v1.ExecuteSqlRequest.QueryMode1 + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryMode` + :param query_mode: Mode governing return of results / query plan. + See: + `QueryMode `_. :type query_options: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Options that are provided for query plan stability. From 6c9a1badfed610a18454137e1b45156872914e7e Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Fri, 17 Apr 2020 13:43:52 +1200 Subject: [PATCH 04/16] fix: pass gRPC config options to gRPC channel creation (#26) * fix: pass gRPC config options to grpc channel creation * regen spanner_grpc_transport.py to include changes (via synth) Co-authored-by: larkee --- .../spanner_v1/gapic/transports/spanner_grpc_transport.py | 3 +++ synth.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py b/google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py index 1a3d0d1407..72b7beeda6 100644 --- a/google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py +++ b/google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py @@ -107,6 +107,9 @@ def create_channel( pkg_resources.resource_string(__name__, _SPANNER_GRPC_CONFIG) ) options = [(grpc_gcp.API_CONFIG_CHANNEL_ARG, grpc_gcp_config)] + if "options" in kwargs: + options.extend(kwargs["options"]) + kwargs["options"] = options return google.api_core.grpc_helpers.create_channel( address, credentials=credentials, scopes=cls._OAUTH_SCOPES, **kwargs ) diff --git a/synth.py b/synth.py index a351bbf943..ed1794b090 100644 --- a/synth.py +++ b/synth.py @@ -62,6 +62,9 @@ "\g<1>grpc_gcp_config = grpc_gcp.api_config_from_text_pb(" "\g<1> pkg_resources.resource_string(__name__, _SPANNER_GRPC_CONFIG))" "\g<1>options = [(grpc_gcp.API_CONFIG_CHANNEL_ARG, grpc_gcp_config)]" + "\g<1>if 'options' in kwargs:" + "\g<1> options.extend(kwargs['options'])" + "\g<1>kwargs['options'] = options" "\g<0>", ) s.replace( From df199f7419ab5419d656300e223d8c1de1d1d18f Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Fri, 17 Apr 2020 15:10:56 +1200 Subject: [PATCH 05/16] test: use assertIsInstance where possible (#57) Co-authored-by: larkee --- tests/unit/test_backup.py | 2 +- tests/unit/test_client.py | 4 ++-- tests/unit/test_database.py | 8 ++++---- tests/unit/test_instance.py | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_backup.py b/tests/unit/test_backup.py index a3b559b763..0762305220 100644 --- a/tests/unit/test_backup.py +++ b/tests/unit/test_backup.py @@ -120,7 +120,7 @@ def test_from_pb_success(self): backup = backup_class.from_pb(backup_pb, instance) - self.assertTrue(isinstance(backup, backup_class)) + self.assertIsInstance(backup, backup_class) self.assertEqual(backup._instance, instance) self.assertEqual(backup.backup_id, self.BACKUP_ID) self.assertEqual(backup._database, "") diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 8308ed6e92..b9446fd867 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -459,7 +459,7 @@ def test_instance_factory_defaults(self): instance = client.instance(self.INSTANCE_ID) - self.assertTrue(isinstance(instance, Instance)) + self.assertIsInstance(instance, Instance) self.assertEqual(instance.instance_id, self.INSTANCE_ID) self.assertIsNone(instance.configuration_name) self.assertEqual(instance.display_name, self.INSTANCE_ID) @@ -479,7 +479,7 @@ def test_instance_factory_explicit(self): node_count=self.NODE_COUNT, ) - self.assertTrue(isinstance(instance, Instance)) + self.assertIsInstance(instance, Instance) self.assertEqual(instance.instance_id, self.INSTANCE_ID) self.assertEqual(instance.configuration_name, self.CONFIGURATION_NAME) self.assertEqual(instance.display_name, self.DISPLAY_NAME) diff --git a/tests/unit/test_database.py b/tests/unit/test_database.py index 4b343c2fd9..37d9eb41a9 100644 --- a/tests/unit/test_database.py +++ b/tests/unit/test_database.py @@ -198,7 +198,7 @@ def test_from_pb_success_w_explicit_pool(self): database = klass.from_pb(database_pb, instance, pool=pool) - self.assertTrue(isinstance(database, klass)) + self.assertIsInstance(database, klass) self.assertEqual(database._instance, instance) self.assertEqual(database.database_id, self.DATABASE_ID) self.assertIs(database._pool, pool) @@ -218,7 +218,7 @@ def test_from_pb_success_w_hyphen_w_default_pool(self): database = klass.from_pb(database_pb, instance) - self.assertTrue(isinstance(database, klass)) + self.assertIsInstance(database, klass) self.assertEqual(database._instance, instance) self.assertEqual(database.database_id, DATABASE_ID_HYPHEN) self.assertIsInstance(database._pool, BurstyPool) @@ -1074,7 +1074,7 @@ def test_session_factory_defaults(self): session = database.session() - self.assertTrue(isinstance(session, Session)) + self.assertIsInstance(session, Session) self.assertIs(session.session_id, None) self.assertIs(session._database, database) self.assertEqual(session.labels, {}) @@ -1090,7 +1090,7 @@ def test_session_factory_w_labels(self): session = database.session(labels=labels) - self.assertTrue(isinstance(session, Session)) + self.assertIsInstance(session, Session) self.assertIs(session.session_id, None) self.assertIs(session._database, database) self.assertEqual(session.labels, labels) diff --git a/tests/unit/test_instance.py b/tests/unit/test_instance.py index b71445d835..c1a0b187ac 100644 --- a/tests/unit/test_instance.py +++ b/tests/unit/test_instance.py @@ -159,7 +159,7 @@ def test_from_pb_success(self): klass = self._getTargetClass() instance = klass.from_pb(instance_pb, client) - self.assertTrue(isinstance(instance, klass)) + self.assertIsInstance(instance, klass) self.assertEqual(instance._client, client) self.assertEqual(instance.instance_id, self.INSTANCE_ID) self.assertEqual(instance.configuration_name, self.CONFIG_NAME) @@ -469,7 +469,7 @@ def test_database_factory_defaults(self): database = instance.database(DATABASE_ID) - self.assertTrue(isinstance(database, Database)) + self.assertIsInstance(database, Database) self.assertEqual(database.database_id, DATABASE_ID) self.assertIs(database._instance, instance) self.assertEqual(list(database.ddl_statements), []) @@ -490,7 +490,7 @@ def test_database_factory_explicit(self): DATABASE_ID, ddl_statements=DDL_STATEMENTS, pool=pool ) - self.assertTrue(isinstance(database, Database)) + self.assertIsInstance(database, Database) self.assertEqual(database.database_id, DATABASE_ID) self.assertIs(database._instance, instance) self.assertEqual(list(database.ddl_statements), DDL_STATEMENTS) From b16ee7c585f16830b1f15d1d7448f287655abba9 Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Tue, 21 Apr 2020 14:33:04 +1200 Subject: [PATCH 06/16] docs: update how to set up background thread to ping PingingPool and TransactionPingingPool (#62) Co-authored-by: larkee --- docs/advanced-session-pool-topics.rst | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/advanced-session-pool-topics.rst b/docs/advanced-session-pool-topics.rst index 18fd7db64c..ae20607ebf 100644 --- a/docs/advanced-session-pool-topics.rst +++ b/docs/advanced-session-pool-topics.rst @@ -57,7 +57,14 @@ from becoming stale: import threading - background = threading.Thread(target=pool.ping, name='ping-pool') + + def background_loop(): + while True: + # (Optional) Perform other background tasks here + pool.ping() + + + background = threading.Thread(target=background_loop, name='ping-pool') background.daemon = True background.start() @@ -91,6 +98,13 @@ started before it is used: import threading - background = threading.Thread(target=pool.ping, name='ping-pool') + + def background_loop(): + while True: + # (Optional) Perform other background tasks here + pool.ping() + + + background = threading.Thread(target=background_loop, name='ping-pool') background.daemon = True background.start() From 1d4976634cb81dd11b0ddc4bfc9fe9c61a7e7041 Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Tue, 21 Apr 2020 18:10:37 +1200 Subject: [PATCH 07/16] docs: add begin_pending_transactions() to the background thread example for TransactionPingingPool (#63) Co-authored-by: larkee --- docs/advanced-session-pool-topics.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/advanced-session-pool-topics.rst b/docs/advanced-session-pool-topics.rst index ae20607ebf..1b21fdcc9b 100644 --- a/docs/advanced-session-pool-topics.rst +++ b/docs/advanced-session-pool-topics.rst @@ -103,6 +103,7 @@ started before it is used: while True: # (Optional) Perform other background tasks here pool.ping() + pool.begin_pending_transactions() background = threading.Thread(target=background_loop, name='ping-pool') From d63dcb68409a1de7cc17bba4afdab54925d427b1 Mon Sep 17 00:00:00 2001 From: Ben Page <8633516+bpg130@users.noreply.github.com> Date: Mon, 27 Apr 2020 23:23:51 -0400 Subject: [PATCH 08/16] docs: make it clear ddl_statements is a list in docs (#59) The ddl_statements fields expects a list of strings, not a string. This aims to make that more clear in the documentation. --- docs/database-usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/database-usage.rst b/docs/database-usage.rst index 8989501a7d..31ecd9908d 100644 --- a/docs/database-usage.rst +++ b/docs/database-usage.rst @@ -29,7 +29,7 @@ To create a :class:`~google.cloud.spanner.database.Database` object: database = instance.database(database_id, ddl_statements) -- ``ddl_statements`` is a string containing DDL for the new database. +- ``ddl_statements`` is a list of strings containing DDL for the new database. You can also use :meth:`Instance.database` to create a local wrapper for a database that has already been created: @@ -68,7 +68,7 @@ via its :meth:`~google.cloud.spanner.database.Database.update_ddl` method: operation = database.update_ddl(ddl_statements, operation_id) -- ``ddl_statements`` is a string containing DDL to be applied to +- ``ddl_statements`` is a list of strings containing DDL to be applied to the database. - ``operation_id`` is a string ID for the long-running operation. From 5d8935d7a799c1324dc3b265a746b11bd53cb5ff Mon Sep 17 00:00:00 2001 From: Vadym Matsishevskyi <25311427+vam-google@users.noreply.github.com> Date: Tue, 28 Apr 2020 15:08:10 -0700 Subject: [PATCH 09/16] chore: Migrate python-spanner synth.py from artman to bazel (#64) --- synth.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/synth.py b/synth.py index ed1794b090..078a866c58 100644 --- a/synth.py +++ b/synth.py @@ -16,17 +16,16 @@ import synthtool as s from synthtool import gcp -gapic = gcp.GAPICGenerator() +gapic = gcp.GAPICBazel() common = gcp.CommonTemplates() # ---------------------------------------------------------------------------- # Generate spanner GAPIC layer # ---------------------------------------------------------------------------- library = gapic.py_library( - "spanner", - "v1", - config_path="/google/spanner/artman_spanner.yaml", - artman_output_name="spanner-v1", + service="spanner", + version="v1", + bazel_target="//google/spanner/v1:spanner-v1-py", include_protos=True, ) @@ -77,10 +76,9 @@ # Generate instance admin client # ---------------------------------------------------------------------------- library = gapic.py_library( - "spanner_admin_instance", - "v1", - config_path="/google/spanner/admin/instance" "/artman_spanner_admin_instance.yaml", - artman_output_name="spanner-admin-instance-v1", + service="spanner_admin_instance", + version="v1", + bazel_target="//google/spanner/admin/instance/v1:admin-instance-v1-py", include_protos=True, ) @@ -111,10 +109,9 @@ # Generate database admin client # ---------------------------------------------------------------------------- library = gapic.py_library( - "spanner_admin_database", - "v1", - config_path="/google/spanner/admin/database" "/artman_spanner_admin_database.yaml", - artman_output_name="spanner-admin-database-v1", + service="spanner_admin_database", + version="v1", + bazel_target="//google/spanner/admin/database/v1:admin-database-v1-py", include_protos=True, ) From 4d54a5cb3b657e67be5e2ae75665776df26c7648 Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Wed, 29 Apr 2020 13:24:08 +1200 Subject: [PATCH 10/16] docs: re-pin sphinx version (#70) Reopens #50 Unfortunately, synthtool has pinned to sphinx<3.0.0 which means any code regen via synth will cause docs to fail. Until it is updated to support it, we also need to pin to sphinx<3.0.0. --- CHANGELOG.md | 2 +- docs/conf.py | 1 - noxfile.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bcc901ac6..c076c35934 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,7 +111,7 @@ Return sessions from pool in LIFO order. ([#9454](https://github.com/googleapis/ - Allow kwargs to be passed to create_channel (via synth). ([#8403](https://github.com/googleapis/google-cloud-python/pull/8403)) ### New Features -- Add 'options_' argument to clients 'get_iam_policy'; pin black version (via synth). ([#8659](https://github.com/googleapis/google-cloud-python/pull/8659)) +- Add 'options\_' argument to clients' 'get_iam_policy'; pin black version (via synth). ([#8659](https://github.com/googleapis/google-cloud-python/pull/8659)) - Add 'client_options' support, update list method docstrings (via synth). ([#8522](https://github.com/googleapis/google-cloud-python/pull/8522)) ### Dependencies diff --git a/docs/conf.py b/docs/conf.py index 4fffc063c8..e326daef4e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,7 +38,6 @@ "sphinx.ext.napoleon", "sphinx.ext.todo", "sphinx.ext.viewcode", - "recommonmark", ] # autodoc/autosummary flags diff --git a/noxfile.py b/noxfile.py index 22f328c4af..ee0e4c8b78 100644 --- a/noxfile.py +++ b/noxfile.py @@ -143,7 +143,7 @@ def docs(session): """Build the docs for this library.""" session.install("-e", ".") - session.install("sphinx", "alabaster", "recommonmark") + session.install("sphinx<3.0.0", "alabaster", "recommonmark") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( From c8c7723956aa8970133d202aa50d9fe0737061ea Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 28 Apr 2020 19:16:07 -0700 Subject: [PATCH 11/16] chore: update backup timeout config (via synth) (#65) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR was generated using Autosynth. :rainbow:
Log from Synthtool ``` 2020-04-21 06:27:51,005 synthtool > Executing /tmpfs/src/git/autosynth/working_repo/synth.py. On branch autosynth nothing to commit, working tree clean 2020-04-21 06:27:51,079 synthtool > Ensuring dependencies. 2020-04-21 06:27:51,084 synthtool > Pulling artman image. latest: Pulling from googleapis/artman Digest: sha256:b3b47805231a305d0f40c4bf069df20f6a2635574e6d4259fac651d3f9f6e098 Status: Image is up to date for googleapis/artman:latest 2020-04-21 06:27:52,770 synthtool > Cloning googleapis. 2020-04-21 06:27:53,424 synthtool > Running generator for google/spanner/artman_spanner.yaml. 2020-04-21 06:28:12,779 synthtool > Generated code into /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-v1. 2020-04-21 06:28:12,780 synthtool > Copy: /home/kbuilder/.cache/synthtool/googleapis/google/spanner/v1/mutation.proto to /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-v1/google/cloud/spanner_v1/proto/mutation.proto 2020-04-21 06:28:12,780 synthtool > Copy: /home/kbuilder/.cache/synthtool/googleapis/google/spanner/v1/spanner.proto to /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-v1/google/cloud/spanner_v1/proto/spanner.proto 2020-04-21 06:28:12,781 synthtool > Copy: /home/kbuilder/.cache/synthtool/googleapis/google/spanner/v1/type.proto to /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-v1/google/cloud/spanner_v1/proto/type.proto 2020-04-21 06:28:12,781 synthtool > Copy: /home/kbuilder/.cache/synthtool/googleapis/google/spanner/v1/keys.proto to /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-v1/google/cloud/spanner_v1/proto/keys.proto 2020-04-21 06:28:12,781 synthtool > Copy: /home/kbuilder/.cache/synthtool/googleapis/google/spanner/v1/result_set.proto to /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-v1/google/cloud/spanner_v1/proto/result_set.proto 2020-04-21 06:28:12,781 synthtool > Copy: /home/kbuilder/.cache/synthtool/googleapis/google/spanner/v1/transaction.proto to /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-v1/google/cloud/spanner_v1/proto/transaction.proto 2020-04-21 06:28:12,782 synthtool > Copy: /home/kbuilder/.cache/synthtool/googleapis/google/spanner/v1/query_plan.proto to /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-v1/google/cloud/spanner_v1/proto/query_plan.proto 2020-04-21 06:28:12,782 synthtool > Placed proto files into /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-v1/google/cloud/spanner_v1/proto. 2020-04-21 06:28:12,791 synthtool > Replaced 'import google.api_core.grpc_helpers\n' in google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py. 2020-04-21 06:28:12,791 synthtool > Replaced 'from google.cloud.spanner_v1.proto import spanner_pb2_grpc\n' in google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py. 2020-04-21 06:28:12,793 synthtool > Replaced "(\\s+)'grpc.max_receive_message_length': -1," in google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py. 2020-04-21 06:28:12,795 synthtool > Replaced '(\\s+)return google.api_core.grpc_helpers.create_channel\\(\n' in google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py. 2020-04-21 06:28:12,795 synthtool > Replaced 'from google.cloud import spanner_v1' in tests/unit/gapic/v1/test_spanner_client_v1.py. 2020-04-21 06:28:12,796 synthtool > Running generator for google/spanner/admin/instance/artman_spanner_admin_instance.yaml. 2020-04-21 06:28:29,994 synthtool > Generated code into /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-admin-instance-v1. 2020-04-21 06:28:29,995 synthtool > Copy: /home/kbuilder/.cache/synthtool/googleapis/google/spanner/admin/instance/v1/spanner_instance_admin.proto to /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-admin-instance-v1/google/cloud/spanner_admin_instance_v1/proto/spanner_instance_admin.proto 2020-04-21 06:28:29,995 synthtool > Placed proto files into /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-admin-instance-v1/google/cloud/spanner_admin_instance_v1/proto. 2020-04-21 06:28:30,000 synthtool > Replaced "'google-cloud-spanner-admin-instance'" in google/cloud/spanner_admin_instance_v1/gapic/instance_admin_client.py. 2020-04-21 06:28:30,004 synthtool > Replaced 'from google\\.cloud\\.spanner\\.admin\\.instance_v1.proto' in google/cloud/spanner_admin_instance_v1/proto/spanner_instance_admin_pb2_grpc.py. 2020-04-21 06:28:30,009 synthtool > Replaced '====*' in google/cloud/spanner_v1/proto/transaction_pb2.py. 2020-04-21 06:28:30,010 synthtool > Replaced '----*' in google/cloud/spanner_v1/proto/transaction_pb2.py. 2020-04-21 06:28:30,010 synthtool > Replaced '~~~~*' in google/cloud/spanner_v1/proto/transaction_pb2.py. 2020-04-21 06:28:30,010 synthtool > Running generator for google/spanner/admin/database/artman_spanner_admin_database.yaml. 2020-04-21 06:28:48,880 synthtool > Generated code into /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-admin-database-v1. 2020-04-21 06:28:48,881 synthtool > Copy: /home/kbuilder/.cache/synthtool/googleapis/google/spanner/admin/database/v1/common.proto to /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-admin-database-v1/google/cloud/spanner_admin_database_v1/proto/common.proto 2020-04-21 06:28:48,881 synthtool > Copy: /home/kbuilder/.cache/synthtool/googleapis/google/spanner/admin/database/v1/backup.proto to /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-admin-database-v1/google/cloud/spanner_admin_database_v1/proto/backup.proto 2020-04-21 06:28:48,882 synthtool > Copy: /home/kbuilder/.cache/synthtool/googleapis/google/spanner/admin/database/v1/spanner_database_admin.proto to /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-admin-database-v1/google/cloud/spanner_admin_database_v1/proto/spanner_database_admin.proto 2020-04-21 06:28:48,882 synthtool > Placed proto files into /home/kbuilder/.cache/synthtool/googleapis/artman-genfiles/python/spanner-admin-database-v1/google/cloud/spanner_admin_database_v1/proto. 2020-04-21 06:28:48,890 synthtool > Replaced "'google-cloud-spanner-admin-database'" in google/cloud/spanner_admin_database_v1/gapic/database_admin_client.py. 2020-04-21 06:28:48,892 synthtool > Replaced 'from google\\.cloud\\.spanner\\.admin\\.database_v1.proto' in google/cloud/spanner_admin_database_v1/proto/backup_pb2.py. 2020-04-21 06:28:48,893 synthtool > Replaced 'from google\\.cloud\\.spanner\\.admin\\.database_v1.proto' in google/cloud/spanner_admin_database_v1/proto/spanner_database_admin_pb2_grpc.py. 2020-04-21 06:28:48,893 synthtool > Replaced 'from google\\.cloud\\.spanner\\.admin\\.database_v1.proto' in google/cloud/spanner_admin_database_v1/proto/spanner_database_admin_pb2.py. 2020-04-21 06:28:48,899 synthtool > Replaced '"""Attributes:' in google/cloud/spanner_admin_instance_v1/proto/spanner_instance_admin_pb2.py. 2020-04-21 06:28:48,900 synthtool > Replaced 'cloud.spanner_admin_instance_v1.types._OperationFuture' in google/cloud/spanner_admin_instance_v1/gapic/instance_admin_client.py. 2020-04-21 06:28:48,901 synthtool > Replaced 'cloud.spanner_admin_database_v1.types._OperationFuture' in google/cloud/spanner_admin_database_v1/gapic/database_admin_client.py. .coveragerc .flake8 .github/CONTRIBUTING.md .github/ISSUE_TEMPLATE/bug_report.md .github/ISSUE_TEMPLATE/feature_request.md .github/ISSUE_TEMPLATE/support_request.md .github/PULL_REQUEST_TEMPLATE.md .github/release-please.yml .gitignore .kokoro/build.sh .kokoro/continuous/common.cfg .kokoro/continuous/continuous.cfg .kokoro/docs/common.cfg .kokoro/docs/docs.cfg .kokoro/presubmit/common.cfg .kokoro/presubmit/presubmit.cfg .kokoro/publish-docs.sh .kokoro/release.sh .kokoro/release/common.cfg .kokoro/release/release.cfg .kokoro/trampoline.sh CODE_OF_CONDUCT.md CONTRIBUTING.rst LICENSE MANIFEST.in docs/_static/custom.css docs/_templates/layout.html docs/conf.py.j2 noxfile.py.j2 renovate.json setup.cfg 2020-04-21 06:28:48,986 synthtool > Replaced 'include README.rst LICENSE\n' in MANIFEST.in. Running session blacken Creating virtual environment (virtualenv) using python3.6 in .nox/blacken pip install black==19.3b0 black docs google tests noxfile.py setup.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_database_v1/gapic/enums.py reformatted /tmpfs/src/git/autosynth/working_repo/docs/conf.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_database_v1/gapic/database_admin_client_config.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_database_v1/proto/backup_pb2_grpc.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_database_v1/gapic/transports/database_admin_grpc_transport.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_database_v1/proto/common_pb2_grpc.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_database_v1/proto/common_pb2.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_database_v1/proto/spanner_database_admin_pb2_grpc.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_instance_v1/gapic/enums.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_database_v1/gapic/database_admin_client.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_instance_v1/gapic/instance_admin_client_config.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_instance_v1/gapic/transports/instance_admin_grpc_transport.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_database_v1/proto/backup_pb2.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_instance_v1/gapic/instance_admin_client.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_instance_v1/proto/spanner_instance_admin_pb2_grpc.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/gapic/enums.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/gapic/spanner_client_config.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_database_v1/proto/spanner_database_admin_pb2.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/keys_pb2_grpc.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/keys_pb2.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/mutation_pb2_grpc.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/gapic/spanner_client.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/query_plan_pb2_grpc.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/mutation_pb2.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/result_set_pb2_grpc.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_admin_instance_v1/proto/spanner_instance_admin_pb2.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/query_plan_pb2.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/result_set_pb2.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/transaction_pb2_grpc.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/spanner_pb2_grpc.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/type_pb2_grpc.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/type_pb2.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/transaction_pb2.py reformatted /tmpfs/src/git/autosynth/working_repo/tests/unit/gapic/v1/test_instance_admin_client_v1.py reformatted /tmpfs/src/git/autosynth/working_repo/tests/unit/gapic/v1/test_database_admin_client_v1.py reformatted /tmpfs/src/git/autosynth/working_repo/tests/unit/gapic/v1/test_spanner_client_v1.py reformatted /tmpfs/src/git/autosynth/working_repo/google/cloud/spanner_v1/proto/spanner_pb2.py All done! ✨ 🍰 ✨ 38 files reformatted, 56 files left unchanged. Session blacken was successful. 2020-04-21 06:29:01,355 synthtool > Wrote metadata to synth.metadata. ```
--- .coveragerc | 16 ++++++ .flake8 | 16 ++++++ .github/ISSUE_TEMPLATE/bug_report.md | 3 +- CONTRIBUTING.rst | 15 ++---- MANIFEST.in | 16 ++++++ .../gapic/database_admin_client_config.py | 6 +-- .../proto/backup.proto | 4 +- .../proto/backup_pb2.py | 51 ++++++++++--------- setup.cfg | 16 ++++++ synth.metadata | 19 ++++--- 10 files changed, 112 insertions(+), 50 deletions(-) diff --git a/.coveragerc b/.coveragerc index b178b094aa..dd39c8546c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,19 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Generated by synthtool. DO NOT EDIT! [run] branch = True diff --git a/.flake8 b/.flake8 index 0268ecc9c5..20fe9bda2e 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,19 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Generated by synthtool. DO NOT EDIT! [flake8] ignore = E203, E266, E501, W503 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 96d9781dc8..2a0c359a3f 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -11,8 +11,7 @@ Thanks for stopping by to let us know something could be better! Please run down the following list and make sure you've tried the usual "quick fixes": - Search the issues already opened: https://github.com/googleapis/python-spanner/issues - - Search the issues on our "catch-all" repository: https://github.com/googleapis/google-cloud-python - - Search StackOverflow: http://stackoverflow.com/questions/tagged/google-cloud-platform+python + - Search StackOverflow: https://stackoverflow.com/questions/tagged/google-cloud-platform+python If you are still having issues, please be sure to include as much information as possible: diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e9fa887ebf..e3b0e9d158 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: 2.7, - 3.5, 3.6, and 3.7 on both UNIX and Windows. + 3.5, 3.6, 3.7 and 3.8 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -214,26 +214,18 @@ We support: - `Python 3.5`_ - `Python 3.6`_ - `Python 3.7`_ +- `Python 3.8`_ .. _Python 3.5: https://docs.python.org/3.5/ .. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ +.. _Python 3.8: https://docs.python.org/3.8/ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-spanner/blob/master/noxfile.py -We explicitly decided not to support `Python 2.5`_ due to `decreased usage`_ -and lack of continuous integration `support`_. - -.. _Python 2.5: https://docs.python.org/2.5/ -.. _decreased usage: https://caremad.io/2013/10/a-look-at-pypi-downloads/ -.. _support: https://blog.travis-ci.com/2013-11-18-upcoming-build-environment-updates/ - -We have `dropped 2.6`_ as a supported version as well since Python 2.6 is no -longer supported by the core development team. - Python 2.7 support is deprecated. All code changes should maintain Python 2.7 compatibility until January 1, 2020. We also explicitly decided to support Python 3 beginning with version @@ -247,7 +239,6 @@ We also explicitly decided to support Python 3 beginning with version .. _prominent: https://docs.djangoproject.com/en/1.9/faq/install/#what-python-version-can-i-use-with-django .. _projects: http://flask.pocoo.org/docs/0.10/python3/ .. _Unicode literal support: https://www.python.org/dev/peps/pep-0414/ -.. _dropped 2.6: https://github.com/googleapis/google-cloud-python/issues/995 ********** Versioning diff --git a/MANIFEST.in b/MANIFEST.in index d96120f55e..b36e3621b0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,19 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Generated by synthtool. DO NOT EDIT! include README.rst LICENSE include google/cloud/spanner_v1/gapic/transports/spanner.grpc.config diff --git a/google/cloud/spanner_admin_database_v1/gapic/database_admin_client_config.py b/google/cloud/spanner_admin_database_v1/gapic/database_admin_client_config.py index d6f830eeee..c82216409b 100644 --- a/google/cloud/spanner_admin_database_v1/gapic/database_admin_client_config.py +++ b/google/cloud/spanner_admin_database_v1/gapic/database_admin_client_config.py @@ -58,7 +58,7 @@ "retry_params_name": "default", }, "CreateBackup": { - "timeout_millis": 30000, + "timeout_millis": 3600000, "retry_codes_name": "non_idempotent", "retry_params_name": "default", }, @@ -73,7 +73,7 @@ "retry_params_name": "default", }, "DeleteBackup": { - "timeout_millis": 30000, + "timeout_millis": 3600000, "retry_codes_name": "idempotent", "retry_params_name": "default", }, @@ -83,7 +83,7 @@ "retry_params_name": "default", }, "RestoreDatabase": { - "timeout_millis": 30000, + "timeout_millis": 3600000, "retry_codes_name": "non_idempotent", "retry_params_name": "default", }, diff --git a/google/cloud/spanner_admin_database_v1/proto/backup.proto b/google/cloud/spanner_admin_database_v1/proto/backup.proto index d9b6fd74cd..b883adf34c 100644 --- a/google/cloud/spanner_admin_database_v1/proto/backup.proto +++ b/google/cloud/spanner_admin_database_v1/proto/backup.proto @@ -56,7 +56,9 @@ message Backup { // created. This needs to be in the same instance as the backup. // Values are of the form // `projects//instances//databases/`. - string database = 2; + string database = 2 [(google.api.resource_reference) = { + type: "spanner.googleapis.com/Database" + }]; // Required for the [CreateBackup][google.spanner.admin.database.v1.DatabaseAdmin.CreateBackup] // operation. The expiration time of the backup, with microseconds diff --git a/google/cloud/spanner_admin_database_v1/proto/backup_pb2.py b/google/cloud/spanner_admin_database_v1/proto/backup_pb2.py index edc596bd94..2d13e69a87 100644 --- a/google/cloud/spanner_admin_database_v1/proto/backup_pb2.py +++ b/google/cloud/spanner_admin_database_v1/proto/backup_pb2.py @@ -36,7 +36,7 @@ "\n$com.google.spanner.admin.database.v1B\013BackupProtoP\001ZHgoogle.golang.org/genproto/googleapis/spanner/admin/database/v1;database\252\002&Google.Cloud.Spanner.Admin.Database.V1\312\002&Google\\Cloud\\Spanner\\Admin\\Database\\V1" ), serialized_pb=_b( - '\n9google/cloud/spanner/admin/database_v1/proto/backup.proto\x12 google.spanner.admin.database.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x19google/api/resource.proto\x1a#google/longrunning/operations.proto\x1a google/protobuf/field_mask.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x39google/cloud/spanner/admin/database_v1/proto/common.proto\x1a\x1cgoogle/api/annotations.proto"\xa7\x03\n\x06\x42\x61\x63kup\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12/\n\x0b\x65xpire_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x34\n\x0b\x63reate_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03\x12\x17\n\nsize_bytes\x18\x05 \x01(\x03\x42\x03\xe0\x41\x03\x12\x42\n\x05state\x18\x06 \x01(\x0e\x32..google.spanner.admin.database.v1.Backup.StateB\x03\xe0\x41\x03\x12"\n\x15referencing_databases\x18\x07 \x03(\tB\x03\xe0\x41\x03"7\n\x05State\x12\x15\n\x11STATE_UNSPECIFIED\x10\x00\x12\x0c\n\x08\x43REATING\x10\x01\x12\t\n\x05READY\x10\x02:\\\xea\x41Y\n\x1dspanner.googleapis.com/Backup\x12\x38projects/{project}/instances/{instance}/backups/{backup}"\xa5\x01\n\x13\x43reateBackupRequest\x12\x37\n\x06parent\x18\x01 \x01(\tB\'\xe0\x41\x02\xfa\x41!\n\x1fspanner.googleapis.com/Instance\x12\x16\n\tbackup_id\x18\x02 \x01(\tB\x03\xe0\x41\x02\x12=\n\x06\x62\x61\x63kup\x18\x03 \x01(\x0b\x32(.google.spanner.admin.database.v1.BackupB\x03\xe0\x41\x02"\xae\x01\n\x14\x43reateBackupMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x45\n\x08progress\x18\x03 \x01(\x0b\x32\x33.google.spanner.admin.database.v1.OperationProgress\x12/\n\x0b\x63\x61ncel_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"\x8a\x01\n\x13UpdateBackupRequest\x12=\n\x06\x62\x61\x63kup\x18\x01 \x01(\x0b\x32(.google.spanner.admin.database.v1.BackupB\x03\xe0\x41\x02\x12\x34\n\x0bupdate_mask\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskB\x03\xe0\x41\x02"G\n\x10GetBackupRequest\x12\x33\n\x04name\x18\x01 \x01(\tB%\xe0\x41\x02\xfa\x41\x1f\n\x1dspanner.googleapis.com/Backup"J\n\x13\x44\x65leteBackupRequest\x12\x33\n\x04name\x18\x01 \x01(\tB%\xe0\x41\x02\xfa\x41\x1f\n\x1dspanner.googleapis.com/Backup"\x84\x01\n\x12ListBackupsRequest\x12\x37\n\x06parent\x18\x01 \x01(\tB\'\xe0\x41\x02\xfa\x41!\n\x1fspanner.googleapis.com/Instance\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\x12\x11\n\tpage_size\x18\x03 \x01(\x05\x12\x12\n\npage_token\x18\x04 \x01(\t"i\n\x13ListBackupsResponse\x12\x39\n\x07\x62\x61\x63kups\x18\x01 \x03(\x0b\x32(.google.spanner.admin.database.v1.Backup\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t"\x8d\x01\n\x1bListBackupOperationsRequest\x12\x37\n\x06parent\x18\x01 \x01(\tB\'\xe0\x41\x02\xfa\x41!\n\x1fspanner.googleapis.com/Instance\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\x12\x11\n\tpage_size\x18\x03 \x01(\x05\x12\x12\n\npage_token\x18\x04 \x01(\t"j\n\x1cListBackupOperationsResponse\x12\x31\n\noperations\x18\x01 \x03(\x0b\x32\x1d.google.longrunning.Operation\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t"f\n\nBackupInfo\x12\x0e\n\x06\x62\x61\x63kup\x18\x01 \x01(\t\x12/\n\x0b\x63reate_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x17\n\x0fsource_database\x18\x03 \x01(\tB\xd1\x01\n$com.google.spanner.admin.database.v1B\x0b\x42\x61\x63kupProtoP\x01ZHgoogle.golang.org/genproto/googleapis/spanner/admin/database/v1;database\xaa\x02&Google.Cloud.Spanner.Admin.Database.V1\xca\x02&Google\\Cloud\\Spanner\\Admin\\Database\\V1b\x06proto3' + '\n9google/cloud/spanner/admin/database_v1/proto/backup.proto\x12 google.spanner.admin.database.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x19google/api/resource.proto\x1a#google/longrunning/operations.proto\x1a google/protobuf/field_mask.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x39google/cloud/spanner/admin/database_v1/proto/common.proto\x1a\x1cgoogle/api/annotations.proto"\xcd\x03\n\x06\x42\x61\x63kup\x12\x36\n\x08\x64\x61tabase\x18\x02 \x01(\tB$\xfa\x41!\n\x1fspanner.googleapis.com/Database\x12/\n\x0b\x65xpire_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x34\n\x0b\x63reate_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03\x12\x17\n\nsize_bytes\x18\x05 \x01(\x03\x42\x03\xe0\x41\x03\x12\x42\n\x05state\x18\x06 \x01(\x0e\x32..google.spanner.admin.database.v1.Backup.StateB\x03\xe0\x41\x03\x12"\n\x15referencing_databases\x18\x07 \x03(\tB\x03\xe0\x41\x03"7\n\x05State\x12\x15\n\x11STATE_UNSPECIFIED\x10\x00\x12\x0c\n\x08\x43REATING\x10\x01\x12\t\n\x05READY\x10\x02:\\\xea\x41Y\n\x1dspanner.googleapis.com/Backup\x12\x38projects/{project}/instances/{instance}/backups/{backup}"\xa5\x01\n\x13\x43reateBackupRequest\x12\x37\n\x06parent\x18\x01 \x01(\tB\'\xe0\x41\x02\xfa\x41!\n\x1fspanner.googleapis.com/Instance\x12\x16\n\tbackup_id\x18\x02 \x01(\tB\x03\xe0\x41\x02\x12=\n\x06\x62\x61\x63kup\x18\x03 \x01(\x0b\x32(.google.spanner.admin.database.v1.BackupB\x03\xe0\x41\x02"\xae\x01\n\x14\x43reateBackupMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x45\n\x08progress\x18\x03 \x01(\x0b\x32\x33.google.spanner.admin.database.v1.OperationProgress\x12/\n\x0b\x63\x61ncel_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"\x8a\x01\n\x13UpdateBackupRequest\x12=\n\x06\x62\x61\x63kup\x18\x01 \x01(\x0b\x32(.google.spanner.admin.database.v1.BackupB\x03\xe0\x41\x02\x12\x34\n\x0bupdate_mask\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskB\x03\xe0\x41\x02"G\n\x10GetBackupRequest\x12\x33\n\x04name\x18\x01 \x01(\tB%\xe0\x41\x02\xfa\x41\x1f\n\x1dspanner.googleapis.com/Backup"J\n\x13\x44\x65leteBackupRequest\x12\x33\n\x04name\x18\x01 \x01(\tB%\xe0\x41\x02\xfa\x41\x1f\n\x1dspanner.googleapis.com/Backup"\x84\x01\n\x12ListBackupsRequest\x12\x37\n\x06parent\x18\x01 \x01(\tB\'\xe0\x41\x02\xfa\x41!\n\x1fspanner.googleapis.com/Instance\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\x12\x11\n\tpage_size\x18\x03 \x01(\x05\x12\x12\n\npage_token\x18\x04 \x01(\t"i\n\x13ListBackupsResponse\x12\x39\n\x07\x62\x61\x63kups\x18\x01 \x03(\x0b\x32(.google.spanner.admin.database.v1.Backup\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t"\x8d\x01\n\x1bListBackupOperationsRequest\x12\x37\n\x06parent\x18\x01 \x01(\tB\'\xe0\x41\x02\xfa\x41!\n\x1fspanner.googleapis.com/Instance\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\x12\x11\n\tpage_size\x18\x03 \x01(\x05\x12\x12\n\npage_token\x18\x04 \x01(\t"j\n\x1cListBackupOperationsResponse\x12\x31\n\noperations\x18\x01 \x03(\x0b\x32\x1d.google.longrunning.Operation\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t"f\n\nBackupInfo\x12\x0e\n\x06\x62\x61\x63kup\x18\x01 \x01(\t\x12/\n\x0b\x63reate_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x17\n\x0fsource_database\x18\x03 \x01(\tB\xd1\x01\n$com.google.spanner.admin.database.v1B\x0b\x42\x61\x63kupProtoP\x01ZHgoogle.golang.org/genproto/googleapis/spanner/admin/database/v1;database\xaa\x02&Google.Cloud.Spanner.Admin.Database.V1\xca\x02&Google\\Cloud\\Spanner\\Admin\\Database\\V1b\x06proto3' ), dependencies=[ google_dot_api_dot_field__behavior__pb2.DESCRIPTOR, @@ -72,8 +72,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=623, - serialized_end=678, + serialized_start=661, + serialized_end=716, ) _sym_db.RegisterEnumDescriptor(_BACKUP_STATE) @@ -100,7 +100,7 @@ containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, + serialized_options=_b("\372A!\n\037spanner.googleapis.com/Database"), file=DESCRIPTOR, ), _descriptor.FieldDescriptor( @@ -223,7 +223,7 @@ extension_ranges=[], oneofs=[], serialized_start=349, - serialized_end=772, + serialized_end=810, ) @@ -299,8 +299,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=775, - serialized_end=940, + serialized_start=813, + serialized_end=978, ) @@ -392,8 +392,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=943, - serialized_end=1117, + serialized_start=981, + serialized_end=1155, ) @@ -449,8 +449,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1120, - serialized_end=1258, + serialized_start=1158, + serialized_end=1296, ) @@ -490,8 +490,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1260, - serialized_end=1331, + serialized_start=1298, + serialized_end=1369, ) @@ -531,8 +531,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1333, - serialized_end=1407, + serialized_start=1371, + serialized_end=1445, ) @@ -626,8 +626,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1410, - serialized_end=1542, + serialized_start=1448, + serialized_end=1580, ) @@ -683,8 +683,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1544, - serialized_end=1649, + serialized_start=1582, + serialized_end=1687, ) @@ -778,8 +778,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1652, - serialized_end=1793, + serialized_start=1690, + serialized_end=1831, ) @@ -835,8 +835,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1795, - serialized_end=1901, + serialized_start=1833, + serialized_end=1939, ) @@ -910,8 +910,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1903, - serialized_end=2005, + serialized_start=1941, + serialized_end=2043, ) _BACKUP.fields_by_name[ @@ -1362,6 +1362,7 @@ DESCRIPTOR._options = None +_BACKUP.fields_by_name["database"]._options = None _BACKUP.fields_by_name["create_time"]._options = None _BACKUP.fields_by_name["size_bytes"]._options = None _BACKUP.fields_by_name["state"]._options = None diff --git a/setup.cfg b/setup.cfg index 3bd555500e..c3a2b39f65 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,19 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Generated by synthtool. DO NOT EDIT! [bdist_wheel] universal = 1 diff --git a/synth.metadata b/synth.metadata index bb226f324a..65874481f1 100644 --- a/synth.metadata +++ b/synth.metadata @@ -1,27 +1,32 @@ { - "updateTime": "2020-03-24T12:17:04.474073Z", "sources": [ { "generator": { "name": "artman", - "version": "1.1.1", - "dockerImage": "googleapis/artman@sha256:5ef340c8d9334719bc5c6981d95f4a5d2737b0a6a24f2b9a0d430e96fff85c5b" + "version": "2.0.0", + "dockerImage": "googleapis/artman@sha256:b3b47805231a305d0f40c4bf069df20f6a2635574e6d4259fac651d3f9f6e098" + } + }, + { + "git": { + "name": ".", + "remote": "https://github.com/googleapis/python-spanner.git", + "sha": "1d4976634cb81dd11b0ddc4bfc9fe9c61a7e7041" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "36c0febd0fa7267ab66d14408eec2afd1b6bec4e", - "internalRef": "302639621", - "log": "36c0febd0fa7267ab66d14408eec2afd1b6bec4e\nUpdate GAPIC configurations to v2 .yaml.\n\nPiperOrigin-RevId: 302639621\n\n078f222366ed344509a48f2f084944ef61476613\nFix containeranalysis v1beta1 assembly target name\n\nPiperOrigin-RevId: 302529186\n\n0be7105dc52590fa9a24e784052298ae37ce53aa\nAdd BUILD.bazel file to asset/v1p1beta1\n\nPiperOrigin-RevId: 302154871\n\n6c248fd13e8543f8d22cbf118d978301a9fbe2a8\nAdd missing resource annotations and additional_bindings to dialogflow v2 API.\n\nPiperOrigin-RevId: 302063117\n\n9a3a7f33be9eeacf7b3e98435816b7022d206bd7\nChange the service name from \"chromeos-moblab.googleapis.com\" to \"chromeosmoblab.googleapis.com\"\n\nPiperOrigin-RevId: 302060989\n\n98a339237577e3de26cb4921f75fb5c57cc7a19f\nfeat: devtools/build/v1 publish client library config annotations\n\n* add details field to some of the BuildEvents\n* add final_invocation_id and build_tool_exit_code fields to BuildStatus\n\nPiperOrigin-RevId: 302044087\n\ncfabc98c6bbbb22d1aeaf7612179c0be193b3a13\nfeat: home/graph/v1 publish client library config annotations & comment updates\n\nThis change includes adding the client library configuration annotations, updated proto comments, and some client library configuration files.\n\nPiperOrigin-RevId: 302042647\n\nc8c8c0bd15d082db9546253dbaad1087c7a9782c\nchore: use latest gapic-generator in bazel WORKSPACE.\nincluding the following commits from gapic-generator:\n- feat: take source protos in all sub-packages (#3144)\n\nPiperOrigin-RevId: 301843591\n\ne4daf5202ea31cb2cb6916fdbfa9d6bd771aeb4c\nAdd bazel file for v1 client lib generation\n\nPiperOrigin-RevId: 301802926\n\n275fbcce2c900278d487c33293a3c7e1fbcd3a34\nfeat: pubsub/v1 add an experimental filter field to Subscription\n\nPiperOrigin-RevId: 301661567\n\nf2b18cec51d27c999ad30011dba17f3965677e9c\nFix: UpdateBackupRequest.backup is a resource, not a resource reference - remove annotation.\n\nPiperOrigin-RevId: 301636171\n\n800384063ac93a0cac3a510d41726fa4b2cd4a83\nCloud Billing Budget API v1beta1\nModified api documentation to include warnings about the new filter field.\n\nPiperOrigin-RevId: 301634389\n\n0cc6c146b660db21f04056c3d58a4b752ee445e3\nCloud Billing Budget API v1alpha1\nModified api documentation to include warnings about the new filter field.\n\nPiperOrigin-RevId: 301630018\n\nff2ea00f69065585c3ac0993c8b582af3b6fc215\nFix: Add resource definition for a parent of InspectTemplate which was otherwise missing.\n\nPiperOrigin-RevId: 301623052\n\n55fa441c9daf03173910760191646399338f2b7c\nAdd proto definition for AccessLevel, AccessPolicy, and ServicePerimeter.\n\nPiperOrigin-RevId: 301620844\n\ne7b10591c5408a67cf14ffafa267556f3290e262\nCloud Bigtable Managed Backup service and message proto files.\n\nPiperOrigin-RevId: 301585144\n\nd8e226f702f8ddf92915128c9f4693b63fb8685d\nfeat: Add time-to-live in a queue for builds\n\nPiperOrigin-RevId: 301579876\n\n430375af011f8c7a5174884f0d0e539c6ffa7675\ndocs: add missing closing backtick\n\nPiperOrigin-RevId: 301538851\n\n0e9f1f60ded9ad1c2e725e37719112f5b487ab65\nbazel: Use latest release of gax_java\n\nPiperOrigin-RevId: 301480457\n\n5058c1c96d0ece7f5301a154cf5a07b2ad03a571\nUpdate GAPIC v2 with batching parameters for Logging API\n\nPiperOrigin-RevId: 301443847\n\n64ab9744073de81fec1b3a6a931befc8a90edf90\nFix: Introduce location-based organization/folder/billing-account resources\nChore: Update copyright years\n\nPiperOrigin-RevId: 301373760\n\n23d5f09e670ebb0c1b36214acf78704e2ecfc2ac\nUpdate field_behavior annotations in V1 and V2.\n\nPiperOrigin-RevId: 301337970\n\nb2cf37e7fd62383a811aa4d54d013ecae638851d\nData Catalog V1 API\n\nPiperOrigin-RevId: 301282503\n\n1976b9981e2900c8172b7d34b4220bdb18c5db42\nCloud DLP api update. Adds missing fields to Finding and adds support for hybrid jobs.\n\nPiperOrigin-RevId: 301205325\n\nae78682c05e864d71223ce22532219813b0245ac\nfix: several sample code blocks in comments are now properly indented for markdown\n\nPiperOrigin-RevId: 301185150\n\ndcd171d04bda5b67db13049320f97eca3ace3731\nPublish Media Translation API V1Beta1\n\nPiperOrigin-RevId: 301180096\n\nff1713453b0fbc5a7544a1ef6828c26ad21a370e\nAdd protos and BUILD rules for v1 API.\n\nPiperOrigin-RevId: 301179394\n\n8386761d09819b665b6a6e1e6d6ff884bc8ff781\nfeat: chromeos/modlab publish protos and config for Chrome OS Moblab API.\n\nPiperOrigin-RevId: 300843960\n\nb2e2bc62fab90e6829e62d3d189906d9b79899e4\nUpdates to GCS gRPC API spec:\n\n1. Changed GetIamPolicy and TestBucketIamPermissions to use wrapper messages around google.iam.v1 IAM requests messages, and added CommonRequestParams. This lets us support RequesterPays buckets.\n2. Added a metadata field to GetObjectMediaResponse, to support resuming an object media read safely (by extracting the generation of the object being read, and using it in the resumed read request).\n\nPiperOrigin-RevId: 300817706\n\n7fd916ce12335cc9e784bb9452a8602d00b2516c\nAdd deprecated_collections field for backward-compatiblity in PHP and monolith-generated Python and Ruby clients.\n\nGenerate TopicName class in Java which covers the functionality of both ProjectTopicName and DeletedTopicName. Introduce breaking changes to be fixed by synth.py.\n\nDelete default retry parameters.\n\nRetry codes defs can be deleted once # https://github.com/googleapis/gapic-generator/issues/3137 is fixed.\n\nPiperOrigin-RevId: 300813135\n\n047d3a8ac7f75383855df0166144f891d7af08d9\nfix!: google/rpc refactor ErrorInfo.type to ErrorInfo.reason and comment updates.\n\nPiperOrigin-RevId: 300773211\n\nfae4bb6d5aac52aabe5f0bb4396466c2304ea6f6\nAdding RetryPolicy to pubsub.proto\n\nPiperOrigin-RevId: 300769420\n\n7d569be2928dbd72b4e261bf9e468f23afd2b950\nAdding additional protocol buffer annotations to v3.\n\nPiperOrigin-RevId: 300718800\n\n13942d1a85a337515040a03c5108993087dc0e4f\nAdd logging protos for Recommender v1.\n\nPiperOrigin-RevId: 300689896\n\na1a573c3eecfe2c404892bfa61a32dd0c9fb22b6\nfix: change go package to use cloud.google.com/go/maps\n\nPiperOrigin-RevId: 300661825\n\nc6fbac11afa0c7ab2972d9df181493875c566f77\nfeat: publish documentai/v1beta2 protos\n\nPiperOrigin-RevId: 300656808\n\n5202a9e0d9903f49e900f20fe5c7f4e42dd6588f\nProtos for v1beta1 release of Cloud Security Center Settings API\n\nPiperOrigin-RevId: 300580858\n\n83518e18655d9d4ac044acbda063cc6ecdb63ef8\nAdds gapic.yaml file and BUILD.bazel file.\n\nPiperOrigin-RevId: 300554200\n\n836c196dc8ef8354bbfb5f30696bd3477e8db5e2\nRegenerate recommender v1beta1 gRPC ServiceConfig file for Insights methods.\n\nPiperOrigin-RevId: 300549302\n\n" + "sha": "42ee97c1b93a0e3759bbba3013da309f670a90ab", + "internalRef": "307114445" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "6a17abc7652e2fe563e1288c6e8c23fc260dda97" + "sha": "f5e4c17dc78a966dbf29961dd01f9bbd63e20a04" } } ], From df4be7fbd84b96592d41ed65e1f6a873ecb39d93 Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Thu, 30 Apr 2020 14:22:03 +1200 Subject: [PATCH 12/16] refactor: remove deprecated resource based routing support (#73) Co-authored-by: larkee --- google/cloud/spanner_v1/client.py | 1 - google/cloud/spanner_v1/database.py | 46 ------ tests/system/test_system.py | 58 ------- tests/unit/test_database.py | 244 +--------------------------- 4 files changed, 4 insertions(+), 345 deletions(-) diff --git a/google/cloud/spanner_v1/client.py b/google/cloud/spanner_v1/client.py index 29f3fcf69a..89ab490cff 100644 --- a/google/cloud/spanner_v1/client.py +++ b/google/cloud/spanner_v1/client.py @@ -158,7 +158,6 @@ class Client(ClientWithProject): _instance_admin_api = None _database_admin_api = None - _endpoint_cache = {} user_agent = None _SET_PROJECT = True # Used by from_service_account_json() diff --git a/google/cloud/spanner_v1/database.py b/google/cloud/spanner_v1/database.py index 99b7244f9d..a3aa3390c4 100644 --- a/google/cloud/spanner_v1/database.py +++ b/google/cloud/spanner_v1/database.py @@ -17,16 +17,12 @@ import copy import functools import grpc -import os import re import threading -import warnings -from google.api_core.client_options import ClientOptions import google.auth.credentials from google.protobuf.struct_pb2 import Struct from google.cloud.exceptions import NotFound -from google.api_core.exceptions import PermissionDenied import six # pylint: disable=ungrouped-imports @@ -67,18 +63,6 @@ _DATABASE_METADATA_FILTER = "name:{0}/operations/" -_RESOURCE_ROUTING_PERMISSIONS_WARNING = ( - "The client library attempted to connect to an endpoint closer to your Cloud Spanner data " - "but was unable to do so. The client library will fall back and route requests to the endpoint " - "given in the client options, which may result in increased latency. " - "We recommend including the scope https://www.googleapis.com/auth/spanner.admin so that the " - "client library can get an instance-specific endpoint and efficiently route requests." -) - - -class ResourceRoutingPermissionsWarning(Warning): - pass - class Database(object): """Representation of a Cloud Spanner Database. @@ -245,36 +229,6 @@ def spanner_api(self): credentials = self._instance._client.credentials if isinstance(credentials, google.auth.credentials.Scoped): credentials = credentials.with_scopes((SPANNER_DATA_SCOPE,)) - if ( - os.getenv("GOOGLE_CLOUD_SPANNER_ENABLE_RESOURCE_BASED_ROUTING") - == "true" - ): - endpoint_cache = self._instance._client._endpoint_cache - if self._instance.name in endpoint_cache: - client_options = ClientOptions( - api_endpoint=endpoint_cache[self._instance.name] - ) - else: - try: - api = self._instance._client.instance_admin_api - resp = api.get_instance( - self._instance.name, - field_mask={"paths": ["endpoint_uris"]}, - metadata=_metadata_with_prefix(self.name), - ) - endpoints = resp.endpoint_uris - if endpoints: - endpoint_cache[self._instance.name] = list(endpoints)[0] - client_options = ClientOptions( - api_endpoint=endpoint_cache[self._instance.name] - ) - # If there are no endpoints, use default endpoint. - except PermissionDenied: - warnings.warn( - _RESOURCE_ROUTING_PERMISSIONS_WARNING, - ResourceRoutingPermissionsWarning, - stacklevel=2, - ) self._spanner_api = SpannerClient( credentials=credentials, client_info=client_info, diff --git a/tests/system/test_system.py b/tests/system/test_system.py index 926cbb4b82..97477119b7 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -56,9 +56,6 @@ CREATE_INSTANCE = os.getenv("GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE") is not None USE_EMULATOR = os.getenv("SPANNER_EMULATOR_HOST") is not None -USE_RESOURCE_ROUTING = ( - os.getenv("GOOGLE_CLOUD_SPANNER_ENABLE_RESOURCE_BASED_ROUTING") == "true" -) if CREATE_INSTANCE: INSTANCE_ID = "google-cloud" + unique_resource_id("-") @@ -286,61 +283,6 @@ def tearDown(self): for doomed in self.to_delete: doomed.drop() - @unittest.skipUnless(USE_RESOURCE_ROUTING, "requires enabling resource routing") - def test_spanner_api_use_user_specified_endpoint(self): - # Clear cache. - Client._endpoint_cache = {} - api = Config.CLIENT.instance_admin_api - resp = api.get_instance( - Config.INSTANCE.name, field_mask={"paths": ["endpoint_uris"]} - ) - if not resp or not resp.endpoint_uris: - return # no resolved endpoint. - resolved_endpoint = resp.endpoint_uris[0] - - client = Client(client_options={"api_endpoint": resolved_endpoint}) - - instance = client.instance(Config.INSTANCE.instance_id) - temp_db_id = "temp_db" + unique_resource_id("_") - temp_db = instance.database(temp_db_id) - temp_db.spanner_api - - # No endpoint cache - Default endpoint used. - self.assertEqual(client._endpoint_cache, {}) - - @unittest.skipUnless(USE_RESOURCE_ROUTING, "requires enabling resource routing") - def test_spanner_api_use_resolved_endpoint(self): - # Clear cache. - Client._endpoint_cache = {} - api = Config.CLIENT.instance_admin_api - resp = api.get_instance( - Config.INSTANCE.name, field_mask={"paths": ["endpoint_uris"]} - ) - if not resp or not resp.endpoint_uris: - return # no resolved endpoint. - resolved_endpoint = resp.endpoint_uris[0] - - client = Client( - client_options=Config.CLIENT._client_options - ) # Use same endpoint as main client. - - instance = client.instance(Config.INSTANCE.instance_id) - temp_db_id = "temp_db" + unique_resource_id("_") - temp_db = instance.database(temp_db_id) - temp_db.spanner_api - - # Endpoint is cached - resolved endpoint used. - self.assertIn(Config.INSTANCE.name, client._endpoint_cache) - self.assertEqual( - client._endpoint_cache[Config.INSTANCE.name], resolved_endpoint - ) - - # Endpoint is cached at a class level. - self.assertIn(Config.INSTANCE.name, Config.CLIENT._endpoint_cache) - self.assertEqual( - Config.CLIENT._endpoint_cache[Config.INSTANCE.name], resolved_endpoint - ) - def test_list_databases(self): # Since `Config.INSTANCE` is newly created in `setUpModule`, the # database created in `setUpClass` here will be the only one. diff --git a/tests/unit/test_database.py b/tests/unit/test_database.py index 37d9eb41a9..5b71b08325 100644 --- a/tests/unit/test_database.py +++ b/tests/unit/test_database.py @@ -260,14 +260,8 @@ def test_restore_info(self): self.assertEqual(database.restore_info, restore_info) def test_spanner_api_property_w_scopeless_creds(self): - from google.cloud.spanner_admin_instance_v1.proto import ( - spanner_instance_admin_pb2 as admin_v1_pb2, - ) client = _Client() - client.instance_admin_api.get_instance.return_value = admin_v1_pb2.Instance( - endpoint_uris=[] - ) client_info = client._client_info = mock.Mock() client_options = client._client_options = mock.Mock() credentials = client.credentials = object() @@ -277,10 +271,8 @@ def test_spanner_api_property_w_scopeless_creds(self): patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with patch as spanner_client: - api = database.spanner_api + with patch as spanner_client: + api = database.spanner_api self.assertIs(api, spanner_client.return_value) @@ -288,7 +280,6 @@ def test_spanner_api_property_w_scopeless_creds(self): again = database.spanner_api self.assertIs(again, api) - client.instance_admin_api.get_instance.assert_called_once() spanner_client.assert_called_once_with( credentials=credentials, client_info=client_info, @@ -297,9 +288,6 @@ def test_spanner_api_property_w_scopeless_creds(self): def test_spanner_api_w_scoped_creds(self): import google.auth.credentials - from google.cloud.spanner_admin_instance_v1.proto import ( - spanner_instance_admin_pb2 as admin_v1_pb2, - ) from google.cloud.spanner_v1.database import SPANNER_DATA_SCOPE class _CredentialsWithScopes(google.auth.credentials.Scoped): @@ -323,22 +311,14 @@ def with_scopes(self, scopes): database = self._make_one(self.DATABASE_ID, instance, pool=pool) patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - client.instance_admin_api.get_instance.return_value = admin_v1_pb2.Instance( - endpoint_uris=[] - ) - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with patch as spanner_client: - api = database.spanner_api - self.assertNotIn(instance.name, client._endpoint_cache) + with patch as spanner_client: + api = database.spanner_api # API instance is cached again = database.spanner_api self.assertIs(again, api) - client.instance_admin_api.get_instance.assert_called_once() self.assertEqual(len(spanner_client.call_args_list), 1) called_args, called_kw = spanner_client.call_args self.assertEqual(called_args, ()) @@ -348,222 +328,6 @@ def with_scopes(self, scopes): self.assertEqual(scoped._scopes, expected_scopes) self.assertIs(scoped._source, credentials) - def test_spanner_api_property_w_scopeless_creds_and_new_endpoint(self): - from google.cloud.spanner_admin_instance_v1.proto import ( - spanner_instance_admin_pb2 as admin_v1_pb2, - ) - - client = _Client() - client.instance_admin_api.get_instance.return_value = admin_v1_pb2.Instance( - endpoint_uris=["test1", "test2"] - ) - client_info = client._client_info = mock.Mock() - client._client_options = mock.Mock() - credentials = client.credentials = object() - instance = _Instance(self.INSTANCE_NAME, client=client) - pool = _Pool() - database = self._make_one(self.DATABASE_ID, instance, pool=pool) - - client_patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - options_patch = mock.patch("google.cloud.spanner_v1.database.ClientOptions") - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with options_patch as options: - with client_patch as spanner_client: - api = database.spanner_api - - self.assertIs(api, spanner_client.return_value) - self.assertIn(instance.name, client._endpoint_cache) - - # API instance is cached - again = database.spanner_api - self.assertIs(again, api) - - self.assertEqual(len(spanner_client.call_args_list), 1) - called_args, called_kw = spanner_client.call_args - self.assertEqual(called_args, ()) - self.assertEqual(called_kw["client_info"], client_info) - self.assertEqual(called_kw["credentials"], credentials) - options.assert_called_with(api_endpoint="test1") - - def test_spanner_api_w_scoped_creds_and_new_endpoint(self): - import google.auth.credentials - from google.cloud.spanner_admin_instance_v1.proto import ( - spanner_instance_admin_pb2 as admin_v1_pb2, - ) - from google.cloud.spanner_v1.database import SPANNER_DATA_SCOPE - - class _CredentialsWithScopes(google.auth.credentials.Scoped): - def __init__(self, scopes=(), source=None): - self._scopes = scopes - self._source = source - - def requires_scopes(self): # pragma: NO COVER - return True - - def with_scopes(self, scopes): - return self.__class__(scopes, self) - - expected_scopes = (SPANNER_DATA_SCOPE,) - client = _Client() - client_info = client._client_info = mock.Mock() - client._client_options = mock.Mock() - credentials = client.credentials = _CredentialsWithScopes() - instance = _Instance(self.INSTANCE_NAME, client=client) - pool = _Pool() - database = self._make_one(self.DATABASE_ID, instance, pool=pool) - - client_patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - options_patch = mock.patch("google.cloud.spanner_v1.database.ClientOptions") - client.instance_admin_api.get_instance.return_value = admin_v1_pb2.Instance( - endpoint_uris=["test1", "test2"] - ) - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with options_patch as options: - with client_patch as spanner_client: - api = database.spanner_api - - self.assertIs(api, spanner_client.return_value) - self.assertIn(instance.name, client._endpoint_cache) - - # API instance is cached - again = database.spanner_api - self.assertIs(again, api) - - self.assertEqual(len(spanner_client.call_args_list), 1) - called_args, called_kw = spanner_client.call_args - self.assertEqual(called_args, ()) - self.assertEqual(called_kw["client_info"], client_info) - scoped = called_kw["credentials"] - self.assertEqual(scoped._scopes, expected_scopes) - self.assertIs(scoped._source, credentials) - options.assert_called_with(api_endpoint="test1") - - def test_spanner_api_resource_routing_permissions_error(self): - from google.api_core.exceptions import PermissionDenied - - client = _Client() - client_info = client._client_info = mock.Mock() - client_options = client._client_options = mock.Mock() - client._endpoint_cache = {} - credentials = client.credentials = mock.Mock() - instance = _Instance(self.INSTANCE_NAME, client=client) - pool = _Pool() - database = self._make_one(self.DATABASE_ID, instance, pool=pool) - - patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - client.instance_admin_api.get_instance.side_effect = PermissionDenied("test") - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with patch as spanner_client: - api = database.spanner_api - - self.assertIs(api, spanner_client.return_value) - - # API instance is cached - again = database.spanner_api - self.assertIs(again, api) - - client.instance_admin_api.get_instance.assert_called_once() - spanner_client.assert_called_once_with( - credentials=credentials, - client_info=client_info, - client_options=client_options, - ) - - def test_spanner_api_disable_resource_routing(self): - client = _Client() - client_info = client._client_info = mock.Mock() - client_options = client._client_options = mock.Mock() - client._endpoint_cache = {} - credentials = client.credentials = mock.Mock() - instance = _Instance(self.INSTANCE_NAME, client=client) - pool = _Pool() - database = self._make_one(self.DATABASE_ID, instance, pool=pool) - - patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "false" - with patch as spanner_client: - api = database.spanner_api - - self.assertIs(api, spanner_client.return_value) - - # API instance is cached - again = database.spanner_api - self.assertIs(again, api) - - client.instance_admin_api.get_instance.assert_not_called() - spanner_client.assert_called_once_with( - credentials=credentials, - client_info=client_info, - client_options=client_options, - ) - - def test_spanner_api_cached_endpoint(self): - from google.cloud.spanner_admin_instance_v1.proto import ( - spanner_instance_admin_pb2 as admin_v1_pb2, - ) - - client = _Client() - client_info = client._client_info = mock.Mock() - client._client_options = mock.Mock() - client._endpoint_cache = {self.INSTANCE_NAME: "cached"} - credentials = client.credentials = mock.Mock() - instance = _Instance(self.INSTANCE_NAME, client=client) - pool = _Pool() - database = self._make_one(self.DATABASE_ID, instance, pool=pool) - - client_patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - options_patch = mock.patch("google.cloud.spanner_v1.database.ClientOptions") - client.instance_admin_api.get_instance.return_value = admin_v1_pb2.Instance( - endpoint_uris=["test1", "test2"] - ) - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with options_patch as options: - with client_patch as spanner_client: - api = database.spanner_api - - self.assertIs(api, spanner_client.return_value) - - # API instance is cached - again = database.spanner_api - self.assertIs(again, api) - - self.assertEqual(len(spanner_client.call_args_list), 1) - called_args, called_kw = spanner_client.call_args - self.assertEqual(called_args, ()) - self.assertEqual(called_kw["client_info"], client_info) - self.assertEqual(called_kw["credentials"], credentials) - options.assert_called_with(api_endpoint="cached") - - def test_spanner_api_resource_routing_error(self): - from google.api_core.exceptions import GoogleAPIError - - client = _Client() - client._client_info = mock.Mock() - client._client_options = mock.Mock() - client.credentials = mock.Mock() - instance = _Instance(self.INSTANCE_NAME, client=client) - pool = _Pool() - database = self._make_one(self.DATABASE_ID, instance, pool=pool) - - client.instance_admin_api.get_instance.side_effect = GoogleAPIError("test") - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with self.assertRaises(GoogleAPIError): - database.spanner_api - - client.instance_admin_api.get_instance.assert_called_once() - def test_spanner_api_w_emulator_host(self): client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client, emulator_host="host") From 8a3d700134a6380c033a879cff0616a648df709b Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Tue, 5 May 2020 10:57:52 +1200 Subject: [PATCH 13/16] feat: add support for retrying aborted partitioned DML statements (#66) * feat: add support for retrying aborted partitioned dml statements * run blacken * use retry settings from config * fix imports from rebase Co-authored-by: larkee --- google/cloud/spanner_v1/database.py | 59 ++++++++++++++++++++--------- tests/unit/test_database.py | 46 +++++++++++++++++++--- 2 files changed, 83 insertions(+), 22 deletions(-) diff --git a/google/cloud/spanner_v1/database.py b/google/cloud/spanner_v1/database.py index a3aa3390c4..e7f6de3724 100644 --- a/google/cloud/spanner_v1/database.py +++ b/google/cloud/spanner_v1/database.py @@ -21,8 +21,10 @@ import threading import google.auth.credentials +from google.api_core.retry import if_exception_type from google.protobuf.struct_pb2 import Struct from google.cloud.exceptions import NotFound +from google.api_core.exceptions import Aborted import six # pylint: disable=ungrouped-imports @@ -394,29 +396,36 @@ def execute_partitioned_dml( metadata = _metadata_with_prefix(self.name) - with SessionCheckout(self._pool) as session: + def execute_pdml(): + with SessionCheckout(self._pool) as session: + + txn = api.begin_transaction( + session.name, txn_options, metadata=metadata + ) - txn = api.begin_transaction(session.name, txn_options, metadata=metadata) + txn_selector = TransactionSelector(id=txn.id) + + restart = functools.partial( + api.execute_streaming_sql, + session.name, + dml, + transaction=txn_selector, + params=params_pb, + param_types=param_types, + query_options=query_options, + metadata=metadata, + ) - txn_selector = TransactionSelector(id=txn.id) + iterator = _restart_on_unavailable(restart) - restart = functools.partial( - api.execute_streaming_sql, - session.name, - dml, - transaction=txn_selector, - params=params_pb, - param_types=param_types, - query_options=query_options, - metadata=metadata, - ) + result_set = StreamedResultSet(iterator) + list(result_set) # consume all partials - iterator = _restart_on_unavailable(restart) + return result_set.stats.row_count_lower_bound - result_set = StreamedResultSet(iterator) - list(result_set) # consume all partials + retry_config = api._method_configs["ExecuteStreamingSql"].retry - return result_set.stats.row_count_lower_bound + return _retry_on_aborted(execute_pdml, retry_config)() def session(self, labels=None): """Factory to create a session for this database. @@ -976,3 +985,19 @@ def __init__(self, source_type, backup_info): @classmethod def from_pb(cls, pb): return cls(pb.source_type, pb.backup_info) + + +def _retry_on_aborted(func, retry_config): + """Helper for :meth:`Database.execute_partitioned_dml`. + + Wrap function in a Retry that will retry on Aborted exceptions + with the retry config specified. + + :type func: callable + :param func: the function to be retried on Aborted exceptions + + :type retry_config: Retry + :param retry_config: retry object with the settings to be used + """ + retry = retry_config.with_predicate(if_exception_type(Aborted)) + return retry(func) diff --git a/tests/unit/test_database.py b/tests/unit/test_database.py index 5b71b08325..d8a581f87b 100644 --- a/tests/unit/test_database.py +++ b/tests/unit/test_database.py @@ -53,6 +53,7 @@ class _BaseTest(unittest.TestCase): SESSION_ID = "session_id" SESSION_NAME = DATABASE_NAME + "/sessions/" + SESSION_ID TRANSACTION_ID = b"transaction_id" + RETRY_TRANSACTION_ID = b"transaction_id_retry" BACKUP_ID = "backup_id" BACKUP_NAME = INSTANCE_NAME + "/backups/" + BACKUP_ID @@ -735,8 +736,10 @@ def test_drop_success(self): ) def _execute_partitioned_dml_helper( - self, dml, params=None, param_types=None, query_options=None + self, dml, params=None, param_types=None, query_options=None, retried=False ): + from google.api_core.exceptions import Aborted + from google.api_core.retry import Retry from google.protobuf.struct_pb2 import Struct from google.cloud.spanner_v1.proto.result_set_pb2 import ( PartialResultSet, @@ -752,6 +755,10 @@ def _execute_partitioned_dml_helper( _merge_query_options, ) + import collections + + MethodConfig = collections.namedtuple("MethodConfig", ["retry"]) + transaction_pb = TransactionPB(id=self.TRANSACTION_ID) stats_pb = ResultSetStats(row_count_lower_bound=2) @@ -765,8 +772,14 @@ def _execute_partitioned_dml_helper( pool.put(session) database = self._make_one(self.DATABASE_ID, instance, pool=pool) api = database._spanner_api = self._make_spanner_api() - api.begin_transaction.return_value = transaction_pb - api.execute_streaming_sql.return_value = iterator + api._method_configs = {"ExecuteStreamingSql": MethodConfig(retry=Retry())} + if retried: + retry_transaction_pb = TransactionPB(id=self.RETRY_TRANSACTION_ID) + api.begin_transaction.side_effect = [transaction_pb, retry_transaction_pb] + api.execute_streaming_sql.side_effect = [Aborted("test"), iterator] + else: + api.begin_transaction.return_value = transaction_pb + api.execute_streaming_sql.return_value = iterator row_count = database.execute_partitioned_dml( dml, params, param_types, query_options @@ -778,11 +791,15 @@ def _execute_partitioned_dml_helper( partitioned_dml=TransactionOptions.PartitionedDml() ) - api.begin_transaction.assert_called_once_with( + api.begin_transaction.assert_called_with( session.name, txn_options, metadata=[("google-cloud-resource-prefix", database.name)], ) + if retried: + self.assertEqual(api.begin_transaction.call_count, 2) + else: + self.assertEqual(api.begin_transaction.call_count, 1) if params: expected_params = Struct( @@ -798,7 +815,7 @@ def _execute_partitioned_dml_helper( expected_query_options, query_options ) - api.execute_streaming_sql.assert_called_once_with( + api.execute_streaming_sql.assert_any_call( self.SESSION_NAME, dml, transaction=expected_transaction, @@ -807,6 +824,22 @@ def _execute_partitioned_dml_helper( query_options=expected_query_options, metadata=[("google-cloud-resource-prefix", database.name)], ) + if retried: + expected_retry_transaction = TransactionSelector( + id=self.RETRY_TRANSACTION_ID + ) + api.execute_streaming_sql.assert_called_with( + self.SESSION_NAME, + dml, + transaction=expected_retry_transaction, + params=expected_params, + param_types=param_types, + query_options=expected_query_options, + metadata=[("google-cloud-resource-prefix", database.name)], + ) + self.assertEqual(api.execute_streaming_sql.call_count, 2) + else: + self.assertEqual(api.execute_streaming_sql.call_count, 1) def test_execute_partitioned_dml_wo_params(self): self._execute_partitioned_dml_helper(dml=DML_WO_PARAM) @@ -828,6 +861,9 @@ def test_execute_partitioned_dml_w_query_options(self): query_options=ExecuteSqlRequest.QueryOptions(optimizer_version="3"), ) + def test_execute_partitioned_dml_wo_params_retry_aborted(self): + self._execute_partitioned_dml_helper(dml=DML_WO_PARAM, retried=True) + def test_session_factory_defaults(self): from google.cloud.spanner_v1.session import Session From 7a07c2b2c0031ec502b00db9067d6f8eb95d701b Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Tue, 5 May 2020 13:14:46 +1200 Subject: [PATCH 14/16] refactor: PingingPool pings sessions using SELECT 1 (#75) Currently, PingingPool pings sessions in the background by calling `session.exists()` which calls `GetSession`. Using `SELECT 1` is preferred and is used in other client libraries such as [Go](https://github.com/googleapis/google-cloud-go/blob/53898305c6f21b3c3eef34fcff6c61a2cb36f602/spanner/session.go#L227): --- google/cloud/spanner_v1/pool.py | 9 ++++- google/cloud/spanner_v1/session.py | 11 ++++++ tests/unit/test_pool.py | 14 +++++-- tests/unit/test_session.py | 60 ++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 5 deletions(-) diff --git a/google/cloud/spanner_v1/pool.py b/google/cloud/spanner_v1/pool.py index cf3413ceb1..2c056fc820 100644 --- a/google/cloud/spanner_v1/pool.py +++ b/google/cloud/spanner_v1/pool.py @@ -314,7 +314,7 @@ class PingingPool(AbstractSessionPool): - Sessions are used in "round-robin" order (LRU first). - "Pings" existing sessions in the background after a specified interval - via an API call (``session.exists()``). + via an API call (``session.ping()``). - Blocks, with a timeout, when :meth:`get` is called on an empty pool. Raises after timing out. @@ -387,6 +387,9 @@ def get(self, timeout=None): # pylint: disable=arguments-differ ping_after, session = self._sessions.get(block=True, timeout=timeout) if _NOW() > ping_after: + # Using session.exists() guarantees the returned session exists. + # session.ping() uses a cached result in the backend which could + # result in a recently deleted session being returned. if not session.exists(): session = self._new_session() session.create() @@ -430,7 +433,9 @@ def ping(self): # Re-add to queue with existing expiration self._sessions.put((ping_after, session)) break - if not session.exists(): # stale + try: + session.ping() + except NotFound: session = self._new_session() session.create() # Re-add to queue with new expiration diff --git a/google/cloud/spanner_v1/session.py b/google/cloud/spanner_v1/session.py index 61e4322012..a84aaa7c6d 100644 --- a/google/cloud/spanner_v1/session.py +++ b/google/cloud/spanner_v1/session.py @@ -153,6 +153,17 @@ def delete(self): api.delete_session(self.name, metadata=metadata) + def ping(self): + """Ping the session to keep it alive by executing "SELECT 1". + + :raises: ValueError: if :attr:`session_id` is not already set. + """ + if self._session_id is None: + raise ValueError("Session ID not set by back-end") + api = self._database.spanner_api + metadata = _metadata_with_prefix(self._database.name) + api.execute_sql(self.name, "SELECT 1", metadata=metadata) + def snapshot(self, **kw): """Create a snapshot to perform a set of reads with shared staleness. diff --git a/tests/unit/test_pool.py b/tests/unit/test_pool.py index b6786a7f0e..6898314955 100644 --- a/tests/unit/test_pool.py +++ b/tests/unit/test_pool.py @@ -567,7 +567,7 @@ def test_ping_oldest_fresh(self): pool.ping() - self.assertFalse(SESSIONS[0]._exists_checked) + self.assertFalse(SESSIONS[0]._pinged) def test_ping_oldest_stale_but_exists(self): import datetime @@ -584,7 +584,7 @@ def test_ping_oldest_stale_but_exists(self): with _Monkey(MUT, _NOW=lambda: later): pool.ping() - self.assertTrue(SESSIONS[0]._exists_checked) + self.assertTrue(SESSIONS[0]._pinged) def test_ping_oldest_stale_and_not_exists(self): import datetime @@ -602,7 +602,7 @@ def test_ping_oldest_stale_and_not_exists(self): with _Monkey(MUT, _NOW=lambda: later): pool.ping() - self.assertTrue(SESSIONS[0]._exists_checked) + self.assertTrue(SESSIONS[0]._pinged) SESSIONS[1].create.assert_called() @@ -850,6 +850,7 @@ def __init__(self, database, exists=True, transaction=None): self._database = database self._exists = exists self._exists_checked = False + self._pinged = False self.create = mock.Mock() self._deleted = False self._transaction = transaction @@ -861,6 +862,13 @@ def exists(self): self._exists_checked = True return self._exists + def ping(self): + from google.cloud.exceptions import NotFound + + self._pinged = True + if not self._exists: + raise NotFound("expired session") + def delete(self): from google.cloud.exceptions import NotFound diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index e2bf18c723..a39c5e9734 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -210,6 +210,66 @@ def test_exists_error(self): metadata=[("google-cloud-resource-prefix", database.name)], ) + def test_ping_wo_session_id(self): + database = self._make_database() + session = self._make_one(database) + with self.assertRaises(ValueError): + session.ping() + + def test_ping_hit(self): + gax_api = self._make_spanner_api() + gax_api.execute_sql.return_value = "1" + database = self._make_database() + database.spanner_api = gax_api + session = self._make_one(database) + session._session_id = self.SESSION_ID + + session.ping() + + gax_api.execute_sql.assert_called_once_with( + self.SESSION_NAME, + "SELECT 1", + metadata=[("google-cloud-resource-prefix", database.name)], + ) + + def test_ping_miss(self): + from google.api_core.exceptions import NotFound + + gax_api = self._make_spanner_api() + gax_api.execute_sql.side_effect = NotFound("testing") + database = self._make_database() + database.spanner_api = gax_api + session = self._make_one(database) + session._session_id = self.SESSION_ID + + with self.assertRaises(NotFound): + session.ping() + + gax_api.execute_sql.assert_called_once_with( + self.SESSION_NAME, + "SELECT 1", + metadata=[("google-cloud-resource-prefix", database.name)], + ) + + def test_ping_error(self): + from google.api_core.exceptions import Unknown + + gax_api = self._make_spanner_api() + gax_api.execute_sql.side_effect = Unknown("testing") + database = self._make_database() + database.spanner_api = gax_api + session = self._make_one(database) + session._session_id = self.SESSION_ID + + with self.assertRaises(Unknown): + session.ping() + + gax_api.execute_sql.assert_called_once_with( + self.SESSION_NAME, + "SELECT 1", + metadata=[("google-cloud-resource-prefix", database.name)], + ) + def test_delete_wo_session_id(self): database = self._make_database() session = self._make_one(database) From b7739da364826b0e413e358778465fbceae59743 Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Tue, 5 May 2020 14:45:08 +1200 Subject: [PATCH 15/16] tests: add backup integration tests (#69) * tests: add backup integration tests * use unique instance ids for restore instances * remove optimization wait and ensure backups are being cleaned up on failures Co-authored-by: larkee --- tests/system/test_system.py | 282 ++++++++++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index 97477119b7..210ab3fecc 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -428,6 +428,288 @@ def _unit_of_work(transaction, name): self.assertEqual(len(rows), 2) +@unittest.skipIf(USE_EMULATOR, "Skipping backup tests") +class TestBackupAPI(unittest.TestCase, _TestData): + DATABASE_NAME = "test_database" + unique_resource_id("_") + DATABASE_NAME_2 = "test_database2" + unique_resource_id("_") + + @classmethod + def setUpClass(cls): + pool = BurstyPool(labels={"testcase": "database_api"}) + db1 = Config.INSTANCE.database( + cls.DATABASE_NAME, ddl_statements=DDL_STATEMENTS, pool=pool + ) + db2 = Config.INSTANCE.database(cls.DATABASE_NAME_2, pool=pool) + cls._db = db1 + cls._dbs = [db1, db2] + op1 = db1.create() + op2 = db2.create() + op1.result(30) # raises on failure / timeout. + op2.result(30) # raises on failure / timeout. + + current_config = Config.INSTANCE.configuration_name + same_config_instance_id = "same-config" + unique_resource_id("-") + cls._same_config_instance = Config.CLIENT.instance( + same_config_instance_id, current_config + ) + op = cls._same_config_instance.create() + op.result(30) + cls._instances = [cls._same_config_instance] + + retry = RetryErrors(exceptions.ServiceUnavailable) + configs = list(retry(Config.CLIENT.list_instance_configs)()) + diff_configs = [ + config.name + for config in configs + if "-us-" in config.name and config.name is not current_config + ] + cls._diff_config_instance = None + if len(diff_configs) > 0: + diff_config_instance_id = "diff-config" + unique_resource_id("-") + cls._diff_config_instance = Config.CLIENT.instance( + diff_config_instance_id, diff_configs[0] + ) + op = cls._diff_config_instance.create() + op.result(30) + cls._instances.append(cls._diff_config_instance) + + @classmethod + def tearDownClass(cls): + for db in cls._dbs: + db.drop() + for instance in cls._instances: + instance.delete() + + def setUp(self): + self.to_delete = [] + self.to_drop = [] + + def tearDown(self): + for doomed in self.to_delete: + doomed.delete() + for doomed in self.to_drop: + doomed.drop() + + def test_create_invalid(self): + from datetime import datetime + from pytz import UTC + + backup_id = "backup_id" + unique_resource_id("_") + expire_time = datetime.utcnow() + expire_time = expire_time.replace(tzinfo=UTC) + + backup = Config.INSTANCE.backup( + backup_id, database=self._db, expire_time=expire_time + ) + + with self.assertRaises(exceptions.InvalidArgument): + op = backup.create() + op.result() + + def test_backup_workflow(self): + from datetime import datetime + from datetime import timedelta + from pytz import UTC + + instance = Config.INSTANCE + backup_id = "backup_id" + unique_resource_id("_") + expire_time = datetime.utcnow() + timedelta(days=3) + expire_time = expire_time.replace(tzinfo=UTC) + + # Create backup. + backup = instance.backup(backup_id, database=self._db, expire_time=expire_time) + operation = backup.create() + self.to_delete.append(backup) + + # Check metadata. + metadata = operation.metadata + self.assertEqual(backup.name, metadata.name) + self.assertEqual(self._db.name, metadata.database) + operation.result() + + # Check backup object. + backup.reload() + self.assertEqual(self._db.name, backup._database) + self.assertEqual(expire_time, backup.expire_time) + self.assertIsNotNone(backup.create_time) + self.assertIsNotNone(backup.size_bytes) + self.assertIsNotNone(backup.state) + + # Update with valid argument. + valid_expire_time = datetime.utcnow() + timedelta(days=7) + valid_expire_time = valid_expire_time.replace(tzinfo=UTC) + backup.update_expire_time(valid_expire_time) + self.assertEqual(valid_expire_time, backup.expire_time) + + # Restore database to same instance. + restored_id = "restored_db" + unique_resource_id("_") + database = instance.database(restored_id) + self.to_drop.append(database) + operation = database.restore(source=backup) + operation.result() + + database.drop() + backup.delete() + self.assertFalse(backup.exists()) + + def test_restore_to_diff_instance(self): + from datetime import datetime + from datetime import timedelta + from pytz import UTC + + backup_id = "backup_id" + unique_resource_id("_") + expire_time = datetime.utcnow() + timedelta(days=3) + expire_time = expire_time.replace(tzinfo=UTC) + + # Create backup. + backup = Config.INSTANCE.backup( + backup_id, database=self._db, expire_time=expire_time + ) + op = backup.create() + self.to_delete.append(backup) + op.result() + + # Restore database to different instance with same config. + restored_id = "restored_db" + unique_resource_id("_") + database = self._same_config_instance.database(restored_id) + self.to_drop.append(database) + operation = database.restore(source=backup) + operation.result() + + database.drop() + backup.delete() + self.assertFalse(backup.exists()) + + def test_multi_create_cancel_update_error_restore_errors(self): + from datetime import datetime + from datetime import timedelta + from pytz import UTC + + backup_id_1 = "backup_id1" + unique_resource_id("_") + backup_id_2 = "backup_id2" + unique_resource_id("_") + + instance = Config.INSTANCE + expire_time = datetime.utcnow() + timedelta(days=3) + expire_time = expire_time.replace(tzinfo=UTC) + + backup1 = instance.backup( + backup_id_1, database=self._dbs[0], expire_time=expire_time + ) + backup2 = instance.backup( + backup_id_2, database=self._dbs[1], expire_time=expire_time + ) + + # Create two backups. + op1 = backup1.create() + op2 = backup2.create() + self.to_delete.extend([backup1, backup2]) + + backup1.reload() + self.assertFalse(backup1.is_ready()) + backup2.reload() + self.assertFalse(backup2.is_ready()) + + # Cancel a create operation. + op2.cancel() + self.assertTrue(op2.cancelled()) + + op1.result() + backup1.reload() + self.assertTrue(backup1.is_ready()) + + # Update expire time to invalid value. + invalid_expire_time = datetime.now() + timedelta(days=366) + invalid_expire_time = invalid_expire_time.replace(tzinfo=UTC) + with self.assertRaises(exceptions.InvalidArgument): + backup1.update_expire_time(invalid_expire_time) + + # Restore to existing database. + with self.assertRaises(exceptions.AlreadyExists): + self._db.restore(source=backup1) + + # Restore to instance with different config. + if self._diff_config_instance is not None: + return + new_db = self._diff_config_instance.database("diff_config") + op = new_db.create() + op.result(30) + self.to_drop.append(new_db) + with self.assertRaises(exceptions.InvalidArgument): + new_db.restore(source=backup1) + + def test_list_backups(self): + from datetime import datetime + from datetime import timedelta + from pytz import UTC + + backup_id_1 = "backup_id1" + unique_resource_id("_") + backup_id_2 = "backup_id2" + unique_resource_id("_") + + instance = Config.INSTANCE + expire_time_1 = datetime.utcnow() + timedelta(days=21) + expire_time_1 = expire_time_1.replace(tzinfo=UTC) + + backup1 = Config.INSTANCE.backup( + backup_id_1, database=self._dbs[0], expire_time=expire_time_1 + ) + + expire_time_2 = datetime.utcnow() + timedelta(days=1) + expire_time_2 = expire_time_2.replace(tzinfo=UTC) + backup2 = Config.INSTANCE.backup( + backup_id_2, database=self._dbs[1], expire_time=expire_time_2 + ) + + # Create two backups. + op1 = backup1.create() + op1.result() + backup1.reload() + create_time_compare = datetime.utcnow().replace(tzinfo=UTC) + + backup2.create() + self.to_delete.extend([backup1, backup2]) + + # List backups filtered by state. + filter_ = "state:CREATING" + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup2.name) + + # List backups filtered by backup name. + filter_ = "name:{0}".format(backup_id_1) + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup1.name) + + # List backups filtered by database name. + filter_ = "database:{0}".format(self._dbs[0].name) + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup1.name) + + # List backups filtered by create time. + filter_ = 'create_time > "{0}"'.format( + create_time_compare.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + ) + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup2.name) + + # List backups filtered by expire time. + filter_ = 'expire_time > "{0}"'.format( + expire_time_1.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + ) + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup1.name) + + # List backups filtered by size bytes. + filter_ = "size_bytes < {0}".format(backup1.size_bytes) + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup2.name) + + # List backups using pagination. + for page in instance.list_backups(page_size=1).pages: + count = 0 + for backup in page: + count += 1 + self.assertEqual(count, 1) + + SOME_DATE = datetime.date(2011, 1, 17) SOME_TIME = datetime.datetime(1989, 1, 17, 17, 59, 12, 345612) NANO_TIME = DatetimeWithNanoseconds(1995, 8, 31, nanosecond=987654321) From a34dc910b5fb080c70751e22f090781527409eb4 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 5 May 2020 15:23:43 +1000 Subject: [PATCH 16/16] chore: release 1.16.0 (#76) * updated CHANGELOG.md [ci skip] * updated setup.cfg [ci skip] * updated setup.py [ci skip] Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: larkee <31196561+larkee@users.noreply.github.com> --- CHANGELOG.md | 13 +++++++++++++ setup.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c076c35934..edf685521d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ [1]: https://pypi.org/project/google-cloud-spanner/#history +## [1.16.0](https://www.github.com/googleapis/python-spanner/compare/v1.15.1...v1.16.0) (2020-05-05) + + +### Features + +* add support for retrying aborted partitioned DML statements ([#66](https://www.github.com/googleapis/python-spanner/issues/66)) ([8a3d700](https://www.github.com/googleapis/python-spanner/commit/8a3d700134a6380c033a879cff0616a648df709b)) + + +### Bug Fixes + +* add keepalive changes to synth.py ([#55](https://www.github.com/googleapis/python-spanner/issues/55)) ([805bbb7](https://www.github.com/googleapis/python-spanner/commit/805bbb766fd9c019f528e2f8ed1379d997622d03)) +* pass gRPC config options to gRPC channel creation ([#26](https://www.github.com/googleapis/python-spanner/issues/26)) ([6c9a1ba](https://www.github.com/googleapis/python-spanner/commit/6c9a1badfed610a18454137e1b45156872914e7e)) + ### [1.15.1](https://www.github.com/googleapis/python-spanner/compare/v1.15.0...v1.15.1) (2020-04-08) diff --git a/setup.py b/setup.py index 911d9c82a1..26f181d371 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ name = "google-cloud-spanner" description = "Cloud Spanner API client library" -version = "1.15.1" +version = "1.16.0" # Should be one of: # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta'