diff --git a/.coveragerc b/.coveragerc index b178b094aa..dd39c8546c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,19 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Generated by synthtool. DO NOT EDIT! [run] branch = True diff --git a/.flake8 b/.flake8 index 0268ecc9c5..20fe9bda2e 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,19 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Generated by synthtool. DO NOT EDIT! [flake8] ignore = E203, E266, E501, W503 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 96d9781dc8..2a0c359a3f 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -11,8 +11,7 @@ Thanks for stopping by to let us know something could be better! Please run down the following list and make sure you've tried the usual "quick fixes": - Search the issues already opened: https://github.com/googleapis/python-spanner/issues - - Search the issues on our "catch-all" repository: https://github.com/googleapis/google-cloud-python - - Search StackOverflow: http://stackoverflow.com/questions/tagged/google-cloud-platform+python + - Search StackOverflow: https://stackoverflow.com/questions/tagged/google-cloud-platform+python If you are still having issues, please be sure to include as much information as possible: diff --git a/CHANGELOG.md b/CHANGELOG.md index 713fe28347..edf685521d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ [1]: https://pypi.org/project/google-cloud-spanner/#history +## [1.16.0](https://www.github.com/googleapis/python-spanner/compare/v1.15.1...v1.16.0) (2020-05-05) + + +### Features + +* add support for retrying aborted partitioned DML statements ([#66](https://www.github.com/googleapis/python-spanner/issues/66)) ([8a3d700](https://www.github.com/googleapis/python-spanner/commit/8a3d700134a6380c033a879cff0616a648df709b)) + + +### Bug Fixes + +* add keepalive changes to synth.py ([#55](https://www.github.com/googleapis/python-spanner/issues/55)) ([805bbb7](https://www.github.com/googleapis/python-spanner/commit/805bbb766fd9c019f528e2f8ed1379d997622d03)) +* pass gRPC config options to gRPC channel creation ([#26](https://www.github.com/googleapis/python-spanner/issues/26)) ([6c9a1ba](https://www.github.com/googleapis/python-spanner/commit/6c9a1badfed610a18454137e1b45156872914e7e)) + ### [1.15.1](https://www.github.com/googleapis/python-spanner/compare/v1.15.0...v1.15.1) (2020-04-08) @@ -106,12 +119,12 @@ Return sessions from pool in LIFO order. ([#9454](https://github.com/googleapis/ ### Implementation Changes -- Add backoff for `run_in_transaction' when backend does not provide 'RetryInfo' in response. ([#8461](https://github.com/googleapis/google-cloud-python/pull/8461)) +- Add backoff for `run_in_transaction` when backend does not provide 'RetryInfo' in response. ([#8461](https://github.com/googleapis/google-cloud-python/pull/8461)) - Adjust gRPC timeouts (via synth). ([#8445](https://github.com/googleapis/google-cloud-python/pull/8445)) - Allow kwargs to be passed to create_channel (via synth). ([#8403](https://github.com/googleapis/google-cloud-python/pull/8403)) ### New Features -- Add 'options_' argument to clients' 'get_iam_policy'; pin black version (via synth). ([#8659](https://github.com/googleapis/google-cloud-python/pull/8659)) +- Add 'options\_' argument to clients' 'get_iam_policy'; pin black version (via synth). ([#8659](https://github.com/googleapis/google-cloud-python/pull/8659)) - Add 'client_options' support, update list method docstrings (via synth). ([#8522](https://github.com/googleapis/google-cloud-python/pull/8522)) ### Dependencies @@ -382,6 +395,6 @@ Return sessions from pool in LIFO order. ([#9454](https://github.com/googleapis/ - Upgrading to `google-cloud-core >= 0.28.0` and adding dependency on `google-api-core` (#4221, #4280) - Deferring to `google-api-core` for `grpcio` and - `googleapis-common-protos`dependencies (#4096, #4098) + `googleapis-common-protos` dependencies (#4096, #4098) PyPI: https://pypi.org/project/google-cloud-spanner/0.29.0/ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e9fa887ebf..e3b0e9d158 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: 2.7, - 3.5, 3.6, and 3.7 on both UNIX and Windows. + 3.5, 3.6, 3.7 and 3.8 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -214,26 +214,18 @@ We support: - `Python 3.5`_ - `Python 3.6`_ - `Python 3.7`_ +- `Python 3.8`_ .. _Python 3.5: https://docs.python.org/3.5/ .. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ +.. _Python 3.8: https://docs.python.org/3.8/ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-spanner/blob/master/noxfile.py -We explicitly decided not to support `Python 2.5`_ due to `decreased usage`_ -and lack of continuous integration `support`_. - -.. _Python 2.5: https://docs.python.org/2.5/ -.. _decreased usage: https://caremad.io/2013/10/a-look-at-pypi-downloads/ -.. _support: https://blog.travis-ci.com/2013-11-18-upcoming-build-environment-updates/ - -We have `dropped 2.6`_ as a supported version as well since Python 2.6 is no -longer supported by the core development team. - Python 2.7 support is deprecated. All code changes should maintain Python 2.7 compatibility until January 1, 2020. We also explicitly decided to support Python 3 beginning with version @@ -247,7 +239,6 @@ We also explicitly decided to support Python 3 beginning with version .. _prominent: https://docs.djangoproject.com/en/1.9/faq/install/#what-python-version-can-i-use-with-django .. _projects: http://flask.pocoo.org/docs/0.10/python3/ .. _Unicode literal support: https://www.python.org/dev/peps/pep-0414/ -.. _dropped 2.6: https://github.com/googleapis/google-cloud-python/issues/995 ********** Versioning diff --git a/MANIFEST.in b/MANIFEST.in index d96120f55e..b36e3621b0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,19 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Generated by synthtool. DO NOT EDIT! include README.rst LICENSE include google/cloud/spanner_v1/gapic/transports/spanner.grpc.config diff --git a/docs/advanced-session-pool-topics.rst b/docs/advanced-session-pool-topics.rst index 18fd7db64c..1b21fdcc9b 100644 --- a/docs/advanced-session-pool-topics.rst +++ b/docs/advanced-session-pool-topics.rst @@ -57,7 +57,14 @@ from becoming stale: import threading - background = threading.Thread(target=pool.ping, name='ping-pool') + + def background_loop(): + while True: + # (Optional) Perform other background tasks here + pool.ping() + + + background = threading.Thread(target=background_loop, name='ping-pool') background.daemon = True background.start() @@ -91,6 +98,14 @@ started before it is used: import threading - background = threading.Thread(target=pool.ping, name='ping-pool') + + def background_loop(): + while True: + # (Optional) Perform other background tasks here + pool.ping() + pool.begin_pending_transactions() + + + background = threading.Thread(target=background_loop, name='ping-pool') background.daemon = True background.start() diff --git a/docs/database-usage.rst b/docs/database-usage.rst index 8989501a7d..31ecd9908d 100644 --- a/docs/database-usage.rst +++ b/docs/database-usage.rst @@ -29,7 +29,7 @@ To create a :class:`~google.cloud.spanner.database.Database` object: database = instance.database(database_id, ddl_statements) -- ``ddl_statements`` is a string containing DDL for the new database. +- ``ddl_statements`` is a list of strings containing DDL for the new database. You can also use :meth:`Instance.database` to create a local wrapper for a database that has already been created: @@ -68,7 +68,7 @@ via its :meth:`~google.cloud.spanner.database.Database.update_ddl` method: operation = database.update_ddl(ddl_statements, operation_id) -- ``ddl_statements`` is a string containing DDL to be applied to +- ``ddl_statements`` is a list of strings containing DDL to be applied to the database. - ``operation_id`` is a string ID for the long-running operation. diff --git a/docs/gapic/v1/admin_database_types.rst b/docs/gapic/v1/admin_database_types.rst index de3d9585c7..fa9aaa73b1 100644 --- a/docs/gapic/v1/admin_database_types.rst +++ b/docs/gapic/v1/admin_database_types.rst @@ -3,3 +3,4 @@ Spanner Admin Database Client Types .. automodule:: google.cloud.spanner_admin_database_v1.types :members: + :noindex: diff --git a/docs/gapic/v1/admin_instance_types.rst b/docs/gapic/v1/admin_instance_types.rst index 4cd06b3ca0..f8f3afa5ff 100644 --- a/docs/gapic/v1/admin_instance_types.rst +++ b/docs/gapic/v1/admin_instance_types.rst @@ -3,3 +3,4 @@ Spanner Admin Instance Client Types .. automodule:: google.cloud.spanner_admin_instance_v1.types :members: + :noindex: diff --git a/docs/gapic/v1/types.rst b/docs/gapic/v1/types.rst index 28956e60c7..54424febf3 100644 --- a/docs/gapic/v1/types.rst +++ b/docs/gapic/v1/types.rst @@ -3,3 +3,4 @@ Spanner Client Types .. automodule:: google.cloud.spanner_v1.types :members: + :noindex: diff --git a/google/cloud/spanner_admin_database_v1/gapic/database_admin_client_config.py b/google/cloud/spanner_admin_database_v1/gapic/database_admin_client_config.py index d6f830eeee..c82216409b 100644 --- a/google/cloud/spanner_admin_database_v1/gapic/database_admin_client_config.py +++ b/google/cloud/spanner_admin_database_v1/gapic/database_admin_client_config.py @@ -58,7 +58,7 @@ "retry_params_name": "default", }, "CreateBackup": { - "timeout_millis": 30000, + "timeout_millis": 3600000, "retry_codes_name": "non_idempotent", "retry_params_name": "default", }, @@ -73,7 +73,7 @@ "retry_params_name": "default", }, "DeleteBackup": { - "timeout_millis": 30000, + "timeout_millis": 3600000, "retry_codes_name": "idempotent", "retry_params_name": "default", }, @@ -83,7 +83,7 @@ "retry_params_name": "default", }, "RestoreDatabase": { - "timeout_millis": 30000, + "timeout_millis": 3600000, "retry_codes_name": "non_idempotent", "retry_params_name": "default", }, diff --git a/google/cloud/spanner_admin_database_v1/proto/backup.proto b/google/cloud/spanner_admin_database_v1/proto/backup.proto index d9b6fd74cd..b883adf34c 100644 --- a/google/cloud/spanner_admin_database_v1/proto/backup.proto +++ b/google/cloud/spanner_admin_database_v1/proto/backup.proto @@ -56,7 +56,9 @@ message Backup { // created. This needs to be in the same instance as the backup. // Values are of the form // `projects//instances//databases/`. - string database = 2; + string database = 2 [(google.api.resource_reference) = { + type: "spanner.googleapis.com/Database" + }]; // Required for the [CreateBackup][google.spanner.admin.database.v1.DatabaseAdmin.CreateBackup] // operation. The expiration time of the backup, with microseconds diff --git a/google/cloud/spanner_admin_database_v1/proto/backup_pb2.py b/google/cloud/spanner_admin_database_v1/proto/backup_pb2.py index edc596bd94..2d13e69a87 100644 --- a/google/cloud/spanner_admin_database_v1/proto/backup_pb2.py +++ b/google/cloud/spanner_admin_database_v1/proto/backup_pb2.py @@ -36,7 +36,7 @@ "\n$com.google.spanner.admin.database.v1B\013BackupProtoP\001ZHgoogle.golang.org/genproto/googleapis/spanner/admin/database/v1;database\252\002&Google.Cloud.Spanner.Admin.Database.V1\312\002&Google\\Cloud\\Spanner\\Admin\\Database\\V1" ), serialized_pb=_b( - '\n9google/cloud/spanner/admin/database_v1/proto/backup.proto\x12 google.spanner.admin.database.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x19google/api/resource.proto\x1a#google/longrunning/operations.proto\x1a google/protobuf/field_mask.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x39google/cloud/spanner/admin/database_v1/proto/common.proto\x1a\x1cgoogle/api/annotations.proto"\xa7\x03\n\x06\x42\x61\x63kup\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12/\n\x0b\x65xpire_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x34\n\x0b\x63reate_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03\x12\x17\n\nsize_bytes\x18\x05 \x01(\x03\x42\x03\xe0\x41\x03\x12\x42\n\x05state\x18\x06 \x01(\x0e\x32..google.spanner.admin.database.v1.Backup.StateB\x03\xe0\x41\x03\x12"\n\x15referencing_databases\x18\x07 \x03(\tB\x03\xe0\x41\x03"7\n\x05State\x12\x15\n\x11STATE_UNSPECIFIED\x10\x00\x12\x0c\n\x08\x43REATING\x10\x01\x12\t\n\x05READY\x10\x02:\\\xea\x41Y\n\x1dspanner.googleapis.com/Backup\x12\x38projects/{project}/instances/{instance}/backups/{backup}"\xa5\x01\n\x13\x43reateBackupRequest\x12\x37\n\x06parent\x18\x01 \x01(\tB\'\xe0\x41\x02\xfa\x41!\n\x1fspanner.googleapis.com/Instance\x12\x16\n\tbackup_id\x18\x02 \x01(\tB\x03\xe0\x41\x02\x12=\n\x06\x62\x61\x63kup\x18\x03 \x01(\x0b\x32(.google.spanner.admin.database.v1.BackupB\x03\xe0\x41\x02"\xae\x01\n\x14\x43reateBackupMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x45\n\x08progress\x18\x03 \x01(\x0b\x32\x33.google.spanner.admin.database.v1.OperationProgress\x12/\n\x0b\x63\x61ncel_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"\x8a\x01\n\x13UpdateBackupRequest\x12=\n\x06\x62\x61\x63kup\x18\x01 \x01(\x0b\x32(.google.spanner.admin.database.v1.BackupB\x03\xe0\x41\x02\x12\x34\n\x0bupdate_mask\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskB\x03\xe0\x41\x02"G\n\x10GetBackupRequest\x12\x33\n\x04name\x18\x01 \x01(\tB%\xe0\x41\x02\xfa\x41\x1f\n\x1dspanner.googleapis.com/Backup"J\n\x13\x44\x65leteBackupRequest\x12\x33\n\x04name\x18\x01 \x01(\tB%\xe0\x41\x02\xfa\x41\x1f\n\x1dspanner.googleapis.com/Backup"\x84\x01\n\x12ListBackupsRequest\x12\x37\n\x06parent\x18\x01 \x01(\tB\'\xe0\x41\x02\xfa\x41!\n\x1fspanner.googleapis.com/Instance\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\x12\x11\n\tpage_size\x18\x03 \x01(\x05\x12\x12\n\npage_token\x18\x04 \x01(\t"i\n\x13ListBackupsResponse\x12\x39\n\x07\x62\x61\x63kups\x18\x01 \x03(\x0b\x32(.google.spanner.admin.database.v1.Backup\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t"\x8d\x01\n\x1bListBackupOperationsRequest\x12\x37\n\x06parent\x18\x01 \x01(\tB\'\xe0\x41\x02\xfa\x41!\n\x1fspanner.googleapis.com/Instance\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\x12\x11\n\tpage_size\x18\x03 \x01(\x05\x12\x12\n\npage_token\x18\x04 \x01(\t"j\n\x1cListBackupOperationsResponse\x12\x31\n\noperations\x18\x01 \x03(\x0b\x32\x1d.google.longrunning.Operation\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t"f\n\nBackupInfo\x12\x0e\n\x06\x62\x61\x63kup\x18\x01 \x01(\t\x12/\n\x0b\x63reate_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x17\n\x0fsource_database\x18\x03 \x01(\tB\xd1\x01\n$com.google.spanner.admin.database.v1B\x0b\x42\x61\x63kupProtoP\x01ZHgoogle.golang.org/genproto/googleapis/spanner/admin/database/v1;database\xaa\x02&Google.Cloud.Spanner.Admin.Database.V1\xca\x02&Google\\Cloud\\Spanner\\Admin\\Database\\V1b\x06proto3' + '\n9google/cloud/spanner/admin/database_v1/proto/backup.proto\x12 google.spanner.admin.database.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x19google/api/resource.proto\x1a#google/longrunning/operations.proto\x1a google/protobuf/field_mask.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x39google/cloud/spanner/admin/database_v1/proto/common.proto\x1a\x1cgoogle/api/annotations.proto"\xcd\x03\n\x06\x42\x61\x63kup\x12\x36\n\x08\x64\x61tabase\x18\x02 \x01(\tB$\xfa\x41!\n\x1fspanner.googleapis.com/Database\x12/\n\x0b\x65xpire_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x34\n\x0b\x63reate_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03\x12\x17\n\nsize_bytes\x18\x05 \x01(\x03\x42\x03\xe0\x41\x03\x12\x42\n\x05state\x18\x06 \x01(\x0e\x32..google.spanner.admin.database.v1.Backup.StateB\x03\xe0\x41\x03\x12"\n\x15referencing_databases\x18\x07 \x03(\tB\x03\xe0\x41\x03"7\n\x05State\x12\x15\n\x11STATE_UNSPECIFIED\x10\x00\x12\x0c\n\x08\x43REATING\x10\x01\x12\t\n\x05READY\x10\x02:\\\xea\x41Y\n\x1dspanner.googleapis.com/Backup\x12\x38projects/{project}/instances/{instance}/backups/{backup}"\xa5\x01\n\x13\x43reateBackupRequest\x12\x37\n\x06parent\x18\x01 \x01(\tB\'\xe0\x41\x02\xfa\x41!\n\x1fspanner.googleapis.com/Instance\x12\x16\n\tbackup_id\x18\x02 \x01(\tB\x03\xe0\x41\x02\x12=\n\x06\x62\x61\x63kup\x18\x03 \x01(\x0b\x32(.google.spanner.admin.database.v1.BackupB\x03\xe0\x41\x02"\xae\x01\n\x14\x43reateBackupMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x45\n\x08progress\x18\x03 \x01(\x0b\x32\x33.google.spanner.admin.database.v1.OperationProgress\x12/\n\x0b\x63\x61ncel_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"\x8a\x01\n\x13UpdateBackupRequest\x12=\n\x06\x62\x61\x63kup\x18\x01 \x01(\x0b\x32(.google.spanner.admin.database.v1.BackupB\x03\xe0\x41\x02\x12\x34\n\x0bupdate_mask\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.FieldMaskB\x03\xe0\x41\x02"G\n\x10GetBackupRequest\x12\x33\n\x04name\x18\x01 \x01(\tB%\xe0\x41\x02\xfa\x41\x1f\n\x1dspanner.googleapis.com/Backup"J\n\x13\x44\x65leteBackupRequest\x12\x33\n\x04name\x18\x01 \x01(\tB%\xe0\x41\x02\xfa\x41\x1f\n\x1dspanner.googleapis.com/Backup"\x84\x01\n\x12ListBackupsRequest\x12\x37\n\x06parent\x18\x01 \x01(\tB\'\xe0\x41\x02\xfa\x41!\n\x1fspanner.googleapis.com/Instance\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\x12\x11\n\tpage_size\x18\x03 \x01(\x05\x12\x12\n\npage_token\x18\x04 \x01(\t"i\n\x13ListBackupsResponse\x12\x39\n\x07\x62\x61\x63kups\x18\x01 \x03(\x0b\x32(.google.spanner.admin.database.v1.Backup\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t"\x8d\x01\n\x1bListBackupOperationsRequest\x12\x37\n\x06parent\x18\x01 \x01(\tB\'\xe0\x41\x02\xfa\x41!\n\x1fspanner.googleapis.com/Instance\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\x12\x11\n\tpage_size\x18\x03 \x01(\x05\x12\x12\n\npage_token\x18\x04 \x01(\t"j\n\x1cListBackupOperationsResponse\x12\x31\n\noperations\x18\x01 \x03(\x0b\x32\x1d.google.longrunning.Operation\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t"f\n\nBackupInfo\x12\x0e\n\x06\x62\x61\x63kup\x18\x01 \x01(\t\x12/\n\x0b\x63reate_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x17\n\x0fsource_database\x18\x03 \x01(\tB\xd1\x01\n$com.google.spanner.admin.database.v1B\x0b\x42\x61\x63kupProtoP\x01ZHgoogle.golang.org/genproto/googleapis/spanner/admin/database/v1;database\xaa\x02&Google.Cloud.Spanner.Admin.Database.V1\xca\x02&Google\\Cloud\\Spanner\\Admin\\Database\\V1b\x06proto3' ), dependencies=[ google_dot_api_dot_field__behavior__pb2.DESCRIPTOR, @@ -72,8 +72,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=623, - serialized_end=678, + serialized_start=661, + serialized_end=716, ) _sym_db.RegisterEnumDescriptor(_BACKUP_STATE) @@ -100,7 +100,7 @@ containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, + serialized_options=_b("\372A!\n\037spanner.googleapis.com/Database"), file=DESCRIPTOR, ), _descriptor.FieldDescriptor( @@ -223,7 +223,7 @@ extension_ranges=[], oneofs=[], serialized_start=349, - serialized_end=772, + serialized_end=810, ) @@ -299,8 +299,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=775, - serialized_end=940, + serialized_start=813, + serialized_end=978, ) @@ -392,8 +392,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=943, - serialized_end=1117, + serialized_start=981, + serialized_end=1155, ) @@ -449,8 +449,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1120, - serialized_end=1258, + serialized_start=1158, + serialized_end=1296, ) @@ -490,8 +490,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1260, - serialized_end=1331, + serialized_start=1298, + serialized_end=1369, ) @@ -531,8 +531,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1333, - serialized_end=1407, + serialized_start=1371, + serialized_end=1445, ) @@ -626,8 +626,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1410, - serialized_end=1542, + serialized_start=1448, + serialized_end=1580, ) @@ -683,8 +683,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1544, - serialized_end=1649, + serialized_start=1582, + serialized_end=1687, ) @@ -778,8 +778,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1652, - serialized_end=1793, + serialized_start=1690, + serialized_end=1831, ) @@ -835,8 +835,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1795, - serialized_end=1901, + serialized_start=1833, + serialized_end=1939, ) @@ -910,8 +910,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=1903, - serialized_end=2005, + serialized_start=1941, + serialized_end=2043, ) _BACKUP.fields_by_name[ @@ -1362,6 +1362,7 @@ DESCRIPTOR._options = None +_BACKUP.fields_by_name["database"]._options = None _BACKUP.fields_by_name["create_time"]._options = None _BACKUP.fields_by_name["size_bytes"]._options = None _BACKUP.fields_by_name["state"]._options = None diff --git a/google/cloud/spanner_v1/client.py b/google/cloud/spanner_v1/client.py index 01b3ddfabf..89ab490cff 100644 --- a/google/cloud/spanner_v1/client.py +++ b/google/cloud/spanner_v1/client.py @@ -127,7 +127,7 @@ class Client(ClientWithProject): If none are specified, the client will attempt to ascertain the credentials from the environment. - :type client_info: :class:`google.api_core.gapic_v1.client_info.ClientInfo` + :type client_info: :class:`~google.api_core.gapic_v1.client_info.ClientInfo` :param client_info: (Optional) The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. Generally, @@ -145,7 +145,7 @@ class Client(ClientWithProject): on the client. API Endpoint should be set through client_options. :type query_options: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Query optimizer configuration to use for the given query. @@ -158,7 +158,6 @@ class Client(ClientWithProject): _instance_admin_api = None _database_admin_api = None - _endpoint_cache = {} user_agent = None _SET_PROJECT = True # Used by from_service_account_json() @@ -341,7 +340,8 @@ def instance( :param configuration_name: (Optional) Name of the instance configuration used to set up the instance's cluster, in the form: - ``projects//instanceConfigs/``. + ``projects//instanceConfigs/`` + ````. **Required** for instances which do not yet exist. :type display_name: str diff --git a/google/cloud/spanner_v1/database.py b/google/cloud/spanner_v1/database.py index 5785953bd7..e7f6de3724 100644 --- a/google/cloud/spanner_v1/database.py +++ b/google/cloud/spanner_v1/database.py @@ -17,16 +17,14 @@ import copy import functools import grpc -import os import re import threading -import warnings -from google.api_core.client_options import ClientOptions import google.auth.credentials +from google.api_core.retry import if_exception_type from google.protobuf.struct_pb2 import Struct from google.cloud.exceptions import NotFound -from google.api_core.exceptions import PermissionDenied +from google.api_core.exceptions import Aborted import six # pylint: disable=ungrouped-imports @@ -67,18 +65,6 @@ _DATABASE_METADATA_FILTER = "name:{0}/operations/" -_RESOURCE_ROUTING_PERMISSIONS_WARNING = ( - "The client library attempted to connect to an endpoint closer to your Cloud Spanner data " - "but was unable to do so. The client library will fall back and route requests to the endpoint " - "given in the client options, which may result in increased latency. " - "We recommend including the scope https://www.googleapis.com/auth/spanner.admin so that the " - "client library can get an instance-specific endpoint and efficiently route requests." -) - - -class ResourceRoutingPermissionsWarning(Warning): - pass - class Database(object): """Representation of a Cloud Spanner Database. @@ -129,7 +115,7 @@ def from_pb(cls, database_pb, instance, pool=None): """Creates an instance of this class from a protobuf. :type database_pb: - :class:`google.spanner.v2.spanner_instance_admin_pb2.Instance` + :class:`~google.spanner.v2.spanner_instance_admin_pb2.Instance` :param database_pb: A instance protobuf object. :type instance: :class:`~google.cloud.spanner_v1.instance.Instance` @@ -245,36 +231,6 @@ def spanner_api(self): credentials = self._instance._client.credentials if isinstance(credentials, google.auth.credentials.Scoped): credentials = credentials.with_scopes((SPANNER_DATA_SCOPE,)) - if ( - os.getenv("GOOGLE_CLOUD_SPANNER_ENABLE_RESOURCE_BASED_ROUTING") - == "true" - ): - endpoint_cache = self._instance._client._endpoint_cache - if self._instance.name in endpoint_cache: - client_options = ClientOptions( - api_endpoint=endpoint_cache[self._instance.name] - ) - else: - try: - api = self._instance._client.instance_admin_api - resp = api.get_instance( - self._instance.name, - field_mask={"paths": ["endpoint_uris"]}, - metadata=_metadata_with_prefix(self.name), - ) - endpoints = resp.endpoint_uris - if endpoints: - endpoint_cache[self._instance.name] = list(endpoints)[0] - client_options = ClientOptions( - api_endpoint=endpoint_cache[self._instance.name] - ) - # If there are no endpoints, use default endpoint. - except PermissionDenied: - warnings.warn( - _RESOURCE_ROUTING_PERMISSIONS_WARNING, - ResourceRoutingPermissionsWarning, - stacklevel=2, - ) self._spanner_api = SpannerClient( credentials=credentials, client_info=client_info, @@ -410,7 +366,7 @@ def execute_partitioned_dml( required if parameters are passed. :type query_options: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Query optimizer configuration to use for the given query. @@ -440,29 +396,36 @@ def execute_partitioned_dml( metadata = _metadata_with_prefix(self.name) - with SessionCheckout(self._pool) as session: + def execute_pdml(): + with SessionCheckout(self._pool) as session: - txn = api.begin_transaction(session.name, txn_options, metadata=metadata) + txn = api.begin_transaction( + session.name, txn_options, metadata=metadata + ) - txn_selector = TransactionSelector(id=txn.id) + txn_selector = TransactionSelector(id=txn.id) + + restart = functools.partial( + api.execute_streaming_sql, + session.name, + dml, + transaction=txn_selector, + params=params_pb, + param_types=param_types, + query_options=query_options, + metadata=metadata, + ) - restart = functools.partial( - api.execute_streaming_sql, - session.name, - dml, - transaction=txn_selector, - params=params_pb, - param_types=param_types, - query_options=query_options, - metadata=metadata, - ) + iterator = _restart_on_unavailable(restart) - iterator = _restart_on_unavailable(restart) + result_set = StreamedResultSet(iterator) + list(result_set) # consume all partials - result_set = StreamedResultSet(iterator) - list(result_set) # consume all partials + return result_set.stats.row_count_lower_bound - return result_set.stats.row_count_lower_bound + retry_config = api._method_configs["ExecuteStreamingSql"].retry + + return _retry_on_aborted(execute_pdml, retry_config)() def session(self, labels=None): """Factory to create a session for this database. @@ -566,7 +529,7 @@ def restore(self, source): :type backup: :class:`~google.cloud.spanner_v1.backup.Backup` :param backup: the path of the backup being restored from. - :rtype: :class:'~google.api_core.operation.Operation` + :rtype: :class:`~google.api_core.operation.Operation` :returns: a future used to poll the status of the create request :raises Conflict: if the database already exists :raises NotFound: @@ -908,7 +871,7 @@ def generate_query_batches( differ. :type query_options: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Query optimizer configuration to use for the given query. @@ -1022,3 +985,19 @@ def __init__(self, source_type, backup_info): @classmethod def from_pb(cls, pb): return cls(pb.source_type, pb.backup_info) + + +def _retry_on_aborted(func, retry_config): + """Helper for :meth:`Database.execute_partitioned_dml`. + + Wrap function in a Retry that will retry on Aborted exceptions + with the retry config specified. + + :type func: callable + :param func: the function to be retried on Aborted exceptions + + :type retry_config: Retry + :param retry_config: retry object with the settings to be used + """ + retry = retry_config.with_predicate(if_exception_type(Aborted)) + return retry(func) diff --git a/google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py b/google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py index 1a3d0d1407..72b7beeda6 100644 --- a/google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py +++ b/google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py @@ -107,6 +107,9 @@ def create_channel( pkg_resources.resource_string(__name__, _SPANNER_GRPC_CONFIG) ) options = [(grpc_gcp.API_CONFIG_CHANNEL_ARG, grpc_gcp_config)] + if "options" in kwargs: + options.extend(kwargs["options"]) + kwargs["options"] = options return google.api_core.grpc_helpers.create_channel( address, credentials=credentials, scopes=cls._OAUTH_SCOPES, **kwargs ) diff --git a/google/cloud/spanner_v1/instance.py b/google/cloud/spanner_v1/instance.py index 4a14032c13..f0809e7d81 100644 --- a/google/cloud/spanner_v1/instance.py +++ b/google/cloud/spanner_v1/instance.py @@ -135,7 +135,7 @@ def from_pb(cls, instance_pb, client): """Creates an instance from a protobuf. :type instance_pb: - :class:`google.spanner.v2.spanner_instance_admin_pb2.Instance` + :class:`~google.spanner.v2.spanner_instance_admin_pb2.Instance` :param instance_pb: A instance protobuf object. :type client: :class:`~google.cloud.spanner_v1.client.Client` @@ -234,7 +234,7 @@ def create(self): before calling :meth:`create`. - :rtype: :class:`google.api_core.operation.Operation` + :rtype: :class:`~google.api_core.operation.Operation` :returns: an operation instance :raises Conflict: if the instance already exists """ diff --git a/google/cloud/spanner_v1/pool.py b/google/cloud/spanner_v1/pool.py index cf3413ceb1..2c056fc820 100644 --- a/google/cloud/spanner_v1/pool.py +++ b/google/cloud/spanner_v1/pool.py @@ -314,7 +314,7 @@ class PingingPool(AbstractSessionPool): - Sessions are used in "round-robin" order (LRU first). - "Pings" existing sessions in the background after a specified interval - via an API call (``session.exists()``). + via an API call (``session.ping()``). - Blocks, with a timeout, when :meth:`get` is called on an empty pool. Raises after timing out. @@ -387,6 +387,9 @@ def get(self, timeout=None): # pylint: disable=arguments-differ ping_after, session = self._sessions.get(block=True, timeout=timeout) if _NOW() > ping_after: + # Using session.exists() guarantees the returned session exists. + # session.ping() uses a cached result in the backend which could + # result in a recently deleted session being returned. if not session.exists(): session = self._new_session() session.create() @@ -430,7 +433,9 @@ def ping(self): # Re-add to queue with existing expiration self._sessions.put((ping_after, session)) break - if not session.exists(): # stale + try: + session.ping() + except NotFound: session = self._new_session() session.create() # Re-add to queue with new expiration diff --git a/google/cloud/spanner_v1/session.py b/google/cloud/spanner_v1/session.py index fc6bb028b7..a84aaa7c6d 100644 --- a/google/cloud/spanner_v1/session.py +++ b/google/cloud/spanner_v1/session.py @@ -153,6 +153,17 @@ def delete(self): api.delete_session(self.name, metadata=metadata) + def ping(self): + """Ping the session to keep it alive by executing "SELECT 1". + + :raises: ValueError: if :attr:`session_id` is not already set. + """ + if self._session_id is None: + raise ValueError("Session ID not set by back-end") + api = self._database.spanner_api + metadata = _metadata_with_prefix(self._database.name) + api.execute_sql(self.name, "SELECT 1", metadata=metadata) + def snapshot(self, **kw): """Create a snapshot to perform a set of reads with shared staleness. @@ -216,18 +227,18 @@ def execute_sql( the names used in ``sql``. :type param_types: - dict, {str -> :class:`google.spanner.v1.type_pb2.TypeCode`} + dict, {str -> :class:`~google.spanner.v1.type_pb2.TypeCode`} :param param_types: (Optional) explicit types for one or more param values; overrides default type detection on the back-end. :type query_mode: - :class:`google.spanner.v1.spanner_pb2.ExecuteSqlRequest.QueryMode` - :param query_mode: Mode governing return of results / query plan. See - https://cloud.google.com/spanner/reference/rpc/google.spanner.v1#google.spanner.v1.ExecuteSqlRequest.QueryMode1 + :class:`~google.spanner.v1.spanner_pb2.ExecuteSqlRequest.QueryMode` + :param query_mode: Mode governing return of results / query plan. See: + `QueryMode `_. :type query_options: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Options that are provided for query plan stability. diff --git a/google/cloud/spanner_v1/snapshot.py b/google/cloud/spanner_v1/snapshot.py index 56b3b6a813..f7b9f07f8f 100644 --- a/google/cloud/spanner_v1/snapshot.py +++ b/google/cloud/spanner_v1/snapshot.py @@ -178,12 +178,13 @@ def execute_sql( required if parameters are passed. :type query_mode: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryMode` - :param query_mode: Mode governing return of results / query plan. See - https://cloud.google.com/spanner/reference/rpc/google.spanner.v1#google.spanner.v1.ExecuteSqlRequest.QueryMode1 + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryMode` + :param query_mode: Mode governing return of results / query plan. + See: + `QueryMode `_. :type query_options: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Query optimizer configuration to use for the given query. diff --git a/google/cloud/spanner_v1/streamed.py b/google/cloud/spanner_v1/streamed.py index 5d1a31e931..dbb4e0dbc0 100644 --- a/google/cloud/spanner_v1/streamed.py +++ b/google/cloud/spanner_v1/streamed.py @@ -32,7 +32,7 @@ class StreamedResultSet(object): :type response_iterator: :param response_iterator: Iterator yielding - :class:`google.cloud.spanner_v1.proto.result_set_pb2.PartialResultSet` + :class:`~google.cloud.spanner_v1.proto.result_set_pb2.PartialResultSet` instances. :type source: :class:`~google.cloud.spanner_v1.snapshot.Snapshot` @@ -195,13 +195,13 @@ def one_or_none(self): class Unmergeable(ValueError): """Unable to merge two values. - :type lhs: :class:`google.protobuf.struct_pb2.Value` + :type lhs: :class:`~google.protobuf.struct_pb2.Value` :param lhs: pending value to be merged - :type rhs: :class:`google.protobuf.struct_pb2.Value` + :type rhs: :class:`~google.protobuf.struct_pb2.Value` :param rhs: remaining value to be merged - :type type_: :class:`google.cloud.spanner_v1.proto.type_pb2.Type` + :type type_: :class:`~google.cloud.spanner_v1.proto.type_pb2.Type` :param type_: field type of values being merged """ diff --git a/google/cloud/spanner_v1/transaction.py b/google/cloud/spanner_v1/transaction.py index 27c260212e..3c1abc7326 100644 --- a/google/cloud/spanner_v1/transaction.py +++ b/google/cloud/spanner_v1/transaction.py @@ -183,12 +183,13 @@ def execute_update( required if parameters are passed. :type query_mode: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryMode` - :param query_mode: Mode governing return of results / query plan. See - https://cloud.google.com/spanner/reference/rpc/google.spanner.v1#google.spanner.v1.ExecuteSqlRequest.QueryMode1 + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryMode` + :param query_mode: Mode governing return of results / query plan. + See: + `QueryMode `_. :type query_options: - :class:`google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` + :class:`~google.cloud.spanner_v1.proto.ExecuteSqlRequest.QueryOptions` or :class:`dict` :param query_options: (Optional) Options that are provided for query plan stability. diff --git a/noxfile.py b/noxfile.py index 88beb02d68..ee0e4c8b78 100644 --- a/noxfile.py +++ b/noxfile.py @@ -143,7 +143,7 @@ def docs(session): """Build the docs for this library.""" session.install("-e", ".") - session.install("sphinx==2.4.4", "alabaster", "recommonmark") + session.install("sphinx<3.0.0", "alabaster", "recommonmark") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( diff --git a/setup.cfg b/setup.cfg index 3bd555500e..c3a2b39f65 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,19 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Generated by synthtool. DO NOT EDIT! [bdist_wheel] universal = 1 diff --git a/setup.py b/setup.py index 911d9c82a1..26f181d371 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ name = "google-cloud-spanner" description = "Cloud Spanner API client library" -version = "1.15.1" +version = "1.16.0" # Should be one of: # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta' diff --git a/synth.metadata b/synth.metadata index bb226f324a..65874481f1 100644 --- a/synth.metadata +++ b/synth.metadata @@ -1,27 +1,32 @@ { - "updateTime": "2020-03-24T12:17:04.474073Z", "sources": [ { "generator": { "name": "artman", - "version": "1.1.1", - "dockerImage": "googleapis/artman@sha256:5ef340c8d9334719bc5c6981d95f4a5d2737b0a6a24f2b9a0d430e96fff85c5b" + "version": "2.0.0", + "dockerImage": "googleapis/artman@sha256:b3b47805231a305d0f40c4bf069df20f6a2635574e6d4259fac651d3f9f6e098" + } + }, + { + "git": { + "name": ".", + "remote": "https://github.com/googleapis/python-spanner.git", + "sha": "1d4976634cb81dd11b0ddc4bfc9fe9c61a7e7041" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "36c0febd0fa7267ab66d14408eec2afd1b6bec4e", - "internalRef": "302639621", - "log": "36c0febd0fa7267ab66d14408eec2afd1b6bec4e\nUpdate GAPIC configurations to v2 .yaml.\n\nPiperOrigin-RevId: 302639621\n\n078f222366ed344509a48f2f084944ef61476613\nFix containeranalysis v1beta1 assembly target name\n\nPiperOrigin-RevId: 302529186\n\n0be7105dc52590fa9a24e784052298ae37ce53aa\nAdd BUILD.bazel file to asset/v1p1beta1\n\nPiperOrigin-RevId: 302154871\n\n6c248fd13e8543f8d22cbf118d978301a9fbe2a8\nAdd missing resource annotations and additional_bindings to dialogflow v2 API.\n\nPiperOrigin-RevId: 302063117\n\n9a3a7f33be9eeacf7b3e98435816b7022d206bd7\nChange the service name from \"chromeos-moblab.googleapis.com\" to \"chromeosmoblab.googleapis.com\"\n\nPiperOrigin-RevId: 302060989\n\n98a339237577e3de26cb4921f75fb5c57cc7a19f\nfeat: devtools/build/v1 publish client library config annotations\n\n* add details field to some of the BuildEvents\n* add final_invocation_id and build_tool_exit_code fields to BuildStatus\n\nPiperOrigin-RevId: 302044087\n\ncfabc98c6bbbb22d1aeaf7612179c0be193b3a13\nfeat: home/graph/v1 publish client library config annotations & comment updates\n\nThis change includes adding the client library configuration annotations, updated proto comments, and some client library configuration files.\n\nPiperOrigin-RevId: 302042647\n\nc8c8c0bd15d082db9546253dbaad1087c7a9782c\nchore: use latest gapic-generator in bazel WORKSPACE.\nincluding the following commits from gapic-generator:\n- feat: take source protos in all sub-packages (#3144)\n\nPiperOrigin-RevId: 301843591\n\ne4daf5202ea31cb2cb6916fdbfa9d6bd771aeb4c\nAdd bazel file for v1 client lib generation\n\nPiperOrigin-RevId: 301802926\n\n275fbcce2c900278d487c33293a3c7e1fbcd3a34\nfeat: pubsub/v1 add an experimental filter field to Subscription\n\nPiperOrigin-RevId: 301661567\n\nf2b18cec51d27c999ad30011dba17f3965677e9c\nFix: UpdateBackupRequest.backup is a resource, not a resource reference - remove annotation.\n\nPiperOrigin-RevId: 301636171\n\n800384063ac93a0cac3a510d41726fa4b2cd4a83\nCloud Billing Budget API v1beta1\nModified api documentation to include warnings about the new filter field.\n\nPiperOrigin-RevId: 301634389\n\n0cc6c146b660db21f04056c3d58a4b752ee445e3\nCloud Billing Budget API v1alpha1\nModified api documentation to include warnings about the new filter field.\n\nPiperOrigin-RevId: 301630018\n\nff2ea00f69065585c3ac0993c8b582af3b6fc215\nFix: Add resource definition for a parent of InspectTemplate which was otherwise missing.\n\nPiperOrigin-RevId: 301623052\n\n55fa441c9daf03173910760191646399338f2b7c\nAdd proto definition for AccessLevel, AccessPolicy, and ServicePerimeter.\n\nPiperOrigin-RevId: 301620844\n\ne7b10591c5408a67cf14ffafa267556f3290e262\nCloud Bigtable Managed Backup service and message proto files.\n\nPiperOrigin-RevId: 301585144\n\nd8e226f702f8ddf92915128c9f4693b63fb8685d\nfeat: Add time-to-live in a queue for builds\n\nPiperOrigin-RevId: 301579876\n\n430375af011f8c7a5174884f0d0e539c6ffa7675\ndocs: add missing closing backtick\n\nPiperOrigin-RevId: 301538851\n\n0e9f1f60ded9ad1c2e725e37719112f5b487ab65\nbazel: Use latest release of gax_java\n\nPiperOrigin-RevId: 301480457\n\n5058c1c96d0ece7f5301a154cf5a07b2ad03a571\nUpdate GAPIC v2 with batching parameters for Logging API\n\nPiperOrigin-RevId: 301443847\n\n64ab9744073de81fec1b3a6a931befc8a90edf90\nFix: Introduce location-based organization/folder/billing-account resources\nChore: Update copyright years\n\nPiperOrigin-RevId: 301373760\n\n23d5f09e670ebb0c1b36214acf78704e2ecfc2ac\nUpdate field_behavior annotations in V1 and V2.\n\nPiperOrigin-RevId: 301337970\n\nb2cf37e7fd62383a811aa4d54d013ecae638851d\nData Catalog V1 API\n\nPiperOrigin-RevId: 301282503\n\n1976b9981e2900c8172b7d34b4220bdb18c5db42\nCloud DLP api update. Adds missing fields to Finding and adds support for hybrid jobs.\n\nPiperOrigin-RevId: 301205325\n\nae78682c05e864d71223ce22532219813b0245ac\nfix: several sample code blocks in comments are now properly indented for markdown\n\nPiperOrigin-RevId: 301185150\n\ndcd171d04bda5b67db13049320f97eca3ace3731\nPublish Media Translation API V1Beta1\n\nPiperOrigin-RevId: 301180096\n\nff1713453b0fbc5a7544a1ef6828c26ad21a370e\nAdd protos and BUILD rules for v1 API.\n\nPiperOrigin-RevId: 301179394\n\n8386761d09819b665b6a6e1e6d6ff884bc8ff781\nfeat: chromeos/modlab publish protos and config for Chrome OS Moblab API.\n\nPiperOrigin-RevId: 300843960\n\nb2e2bc62fab90e6829e62d3d189906d9b79899e4\nUpdates to GCS gRPC API spec:\n\n1. Changed GetIamPolicy and TestBucketIamPermissions to use wrapper messages around google.iam.v1 IAM requests messages, and added CommonRequestParams. This lets us support RequesterPays buckets.\n2. Added a metadata field to GetObjectMediaResponse, to support resuming an object media read safely (by extracting the generation of the object being read, and using it in the resumed read request).\n\nPiperOrigin-RevId: 300817706\n\n7fd916ce12335cc9e784bb9452a8602d00b2516c\nAdd deprecated_collections field for backward-compatiblity in PHP and monolith-generated Python and Ruby clients.\n\nGenerate TopicName class in Java which covers the functionality of both ProjectTopicName and DeletedTopicName. Introduce breaking changes to be fixed by synth.py.\n\nDelete default retry parameters.\n\nRetry codes defs can be deleted once # https://github.com/googleapis/gapic-generator/issues/3137 is fixed.\n\nPiperOrigin-RevId: 300813135\n\n047d3a8ac7f75383855df0166144f891d7af08d9\nfix!: google/rpc refactor ErrorInfo.type to ErrorInfo.reason and comment updates.\n\nPiperOrigin-RevId: 300773211\n\nfae4bb6d5aac52aabe5f0bb4396466c2304ea6f6\nAdding RetryPolicy to pubsub.proto\n\nPiperOrigin-RevId: 300769420\n\n7d569be2928dbd72b4e261bf9e468f23afd2b950\nAdding additional protocol buffer annotations to v3.\n\nPiperOrigin-RevId: 300718800\n\n13942d1a85a337515040a03c5108993087dc0e4f\nAdd logging protos for Recommender v1.\n\nPiperOrigin-RevId: 300689896\n\na1a573c3eecfe2c404892bfa61a32dd0c9fb22b6\nfix: change go package to use cloud.google.com/go/maps\n\nPiperOrigin-RevId: 300661825\n\nc6fbac11afa0c7ab2972d9df181493875c566f77\nfeat: publish documentai/v1beta2 protos\n\nPiperOrigin-RevId: 300656808\n\n5202a9e0d9903f49e900f20fe5c7f4e42dd6588f\nProtos for v1beta1 release of Cloud Security Center Settings API\n\nPiperOrigin-RevId: 300580858\n\n83518e18655d9d4ac044acbda063cc6ecdb63ef8\nAdds gapic.yaml file and BUILD.bazel file.\n\nPiperOrigin-RevId: 300554200\n\n836c196dc8ef8354bbfb5f30696bd3477e8db5e2\nRegenerate recommender v1beta1 gRPC ServiceConfig file for Insights methods.\n\nPiperOrigin-RevId: 300549302\n\n" + "sha": "42ee97c1b93a0e3759bbba3013da309f670a90ab", + "internalRef": "307114445" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "6a17abc7652e2fe563e1288c6e8c23fc260dda97" + "sha": "f5e4c17dc78a966dbf29961dd01f9bbd63e20a04" } } ], diff --git a/synth.py b/synth.py index c509089401..078a866c58 100644 --- a/synth.py +++ b/synth.py @@ -16,17 +16,16 @@ import synthtool as s from synthtool import gcp -gapic = gcp.GAPICGenerator() +gapic = gcp.GAPICBazel() common = gcp.CommonTemplates() # ---------------------------------------------------------------------------- # Generate spanner GAPIC layer # ---------------------------------------------------------------------------- library = gapic.py_library( - "spanner", - "v1", - config_path="/google/spanner/artman_spanner.yaml", - artman_output_name="spanner-v1", + service="spanner", + version="v1", + bazel_target="//google/spanner/v1:spanner-v1-py", include_protos=True, ) @@ -46,7 +45,14 @@ s.replace( "google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py", "from google.cloud.spanner_v1.proto import spanner_pb2_grpc\n", - "\g<0>\n\n_SPANNER_GRPC_CONFIG = 'spanner.grpc.config'\n", + "\g<0>\n\n_GRPC_KEEPALIVE_MS = 2 * 60 * 1000\n" + "_SPANNER_GRPC_CONFIG = 'spanner.grpc.config'\n", +) + +s.replace( + "google/cloud/spanner_v1/gapic/transports/spanner_grpc_transport.py", + "(\s+)'grpc.max_receive_message_length': -1,", + "\g<0>\g<1>\"grpc.keepalive_time_ms\": _GRPC_KEEPALIVE_MS,", ) s.replace( @@ -55,6 +61,9 @@ "\g<1>grpc_gcp_config = grpc_gcp.api_config_from_text_pb(" "\g<1> pkg_resources.resource_string(__name__, _SPANNER_GRPC_CONFIG))" "\g<1>options = [(grpc_gcp.API_CONFIG_CHANNEL_ARG, grpc_gcp_config)]" + "\g<1>if 'options' in kwargs:" + "\g<1> options.extend(kwargs['options'])" + "\g<1>kwargs['options'] = options" "\g<0>", ) s.replace( @@ -67,10 +76,9 @@ # Generate instance admin client # ---------------------------------------------------------------------------- library = gapic.py_library( - "spanner_admin_instance", - "v1", - config_path="/google/spanner/admin/instance" "/artman_spanner_admin_instance.yaml", - artman_output_name="spanner-admin-instance-v1", + service="spanner_admin_instance", + version="v1", + bazel_target="//google/spanner/admin/instance/v1:admin-instance-v1-py", include_protos=True, ) @@ -101,10 +109,9 @@ # Generate database admin client # ---------------------------------------------------------------------------- library = gapic.py_library( - "spanner_admin_database", - "v1", - config_path="/google/spanner/admin/database" "/artman_spanner_admin_database.yaml", - artman_output_name="spanner-admin-database-v1", + service="spanner_admin_database", + version="v1", + bazel_target="//google/spanner/admin/database/v1:admin-database-v1-py", include_protos=True, ) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index 926cbb4b82..210ab3fecc 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -56,9 +56,6 @@ CREATE_INSTANCE = os.getenv("GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE") is not None USE_EMULATOR = os.getenv("SPANNER_EMULATOR_HOST") is not None -USE_RESOURCE_ROUTING = ( - os.getenv("GOOGLE_CLOUD_SPANNER_ENABLE_RESOURCE_BASED_ROUTING") == "true" -) if CREATE_INSTANCE: INSTANCE_ID = "google-cloud" + unique_resource_id("-") @@ -286,61 +283,6 @@ def tearDown(self): for doomed in self.to_delete: doomed.drop() - @unittest.skipUnless(USE_RESOURCE_ROUTING, "requires enabling resource routing") - def test_spanner_api_use_user_specified_endpoint(self): - # Clear cache. - Client._endpoint_cache = {} - api = Config.CLIENT.instance_admin_api - resp = api.get_instance( - Config.INSTANCE.name, field_mask={"paths": ["endpoint_uris"]} - ) - if not resp or not resp.endpoint_uris: - return # no resolved endpoint. - resolved_endpoint = resp.endpoint_uris[0] - - client = Client(client_options={"api_endpoint": resolved_endpoint}) - - instance = client.instance(Config.INSTANCE.instance_id) - temp_db_id = "temp_db" + unique_resource_id("_") - temp_db = instance.database(temp_db_id) - temp_db.spanner_api - - # No endpoint cache - Default endpoint used. - self.assertEqual(client._endpoint_cache, {}) - - @unittest.skipUnless(USE_RESOURCE_ROUTING, "requires enabling resource routing") - def test_spanner_api_use_resolved_endpoint(self): - # Clear cache. - Client._endpoint_cache = {} - api = Config.CLIENT.instance_admin_api - resp = api.get_instance( - Config.INSTANCE.name, field_mask={"paths": ["endpoint_uris"]} - ) - if not resp or not resp.endpoint_uris: - return # no resolved endpoint. - resolved_endpoint = resp.endpoint_uris[0] - - client = Client( - client_options=Config.CLIENT._client_options - ) # Use same endpoint as main client. - - instance = client.instance(Config.INSTANCE.instance_id) - temp_db_id = "temp_db" + unique_resource_id("_") - temp_db = instance.database(temp_db_id) - temp_db.spanner_api - - # Endpoint is cached - resolved endpoint used. - self.assertIn(Config.INSTANCE.name, client._endpoint_cache) - self.assertEqual( - client._endpoint_cache[Config.INSTANCE.name], resolved_endpoint - ) - - # Endpoint is cached at a class level. - self.assertIn(Config.INSTANCE.name, Config.CLIENT._endpoint_cache) - self.assertEqual( - Config.CLIENT._endpoint_cache[Config.INSTANCE.name], resolved_endpoint - ) - def test_list_databases(self): # Since `Config.INSTANCE` is newly created in `setUpModule`, the # database created in `setUpClass` here will be the only one. @@ -486,6 +428,288 @@ def _unit_of_work(transaction, name): self.assertEqual(len(rows), 2) +@unittest.skipIf(USE_EMULATOR, "Skipping backup tests") +class TestBackupAPI(unittest.TestCase, _TestData): + DATABASE_NAME = "test_database" + unique_resource_id("_") + DATABASE_NAME_2 = "test_database2" + unique_resource_id("_") + + @classmethod + def setUpClass(cls): + pool = BurstyPool(labels={"testcase": "database_api"}) + db1 = Config.INSTANCE.database( + cls.DATABASE_NAME, ddl_statements=DDL_STATEMENTS, pool=pool + ) + db2 = Config.INSTANCE.database(cls.DATABASE_NAME_2, pool=pool) + cls._db = db1 + cls._dbs = [db1, db2] + op1 = db1.create() + op2 = db2.create() + op1.result(30) # raises on failure / timeout. + op2.result(30) # raises on failure / timeout. + + current_config = Config.INSTANCE.configuration_name + same_config_instance_id = "same-config" + unique_resource_id("-") + cls._same_config_instance = Config.CLIENT.instance( + same_config_instance_id, current_config + ) + op = cls._same_config_instance.create() + op.result(30) + cls._instances = [cls._same_config_instance] + + retry = RetryErrors(exceptions.ServiceUnavailable) + configs = list(retry(Config.CLIENT.list_instance_configs)()) + diff_configs = [ + config.name + for config in configs + if "-us-" in config.name and config.name is not current_config + ] + cls._diff_config_instance = None + if len(diff_configs) > 0: + diff_config_instance_id = "diff-config" + unique_resource_id("-") + cls._diff_config_instance = Config.CLIENT.instance( + diff_config_instance_id, diff_configs[0] + ) + op = cls._diff_config_instance.create() + op.result(30) + cls._instances.append(cls._diff_config_instance) + + @classmethod + def tearDownClass(cls): + for db in cls._dbs: + db.drop() + for instance in cls._instances: + instance.delete() + + def setUp(self): + self.to_delete = [] + self.to_drop = [] + + def tearDown(self): + for doomed in self.to_delete: + doomed.delete() + for doomed in self.to_drop: + doomed.drop() + + def test_create_invalid(self): + from datetime import datetime + from pytz import UTC + + backup_id = "backup_id" + unique_resource_id("_") + expire_time = datetime.utcnow() + expire_time = expire_time.replace(tzinfo=UTC) + + backup = Config.INSTANCE.backup( + backup_id, database=self._db, expire_time=expire_time + ) + + with self.assertRaises(exceptions.InvalidArgument): + op = backup.create() + op.result() + + def test_backup_workflow(self): + from datetime import datetime + from datetime import timedelta + from pytz import UTC + + instance = Config.INSTANCE + backup_id = "backup_id" + unique_resource_id("_") + expire_time = datetime.utcnow() + timedelta(days=3) + expire_time = expire_time.replace(tzinfo=UTC) + + # Create backup. + backup = instance.backup(backup_id, database=self._db, expire_time=expire_time) + operation = backup.create() + self.to_delete.append(backup) + + # Check metadata. + metadata = operation.metadata + self.assertEqual(backup.name, metadata.name) + self.assertEqual(self._db.name, metadata.database) + operation.result() + + # Check backup object. + backup.reload() + self.assertEqual(self._db.name, backup._database) + self.assertEqual(expire_time, backup.expire_time) + self.assertIsNotNone(backup.create_time) + self.assertIsNotNone(backup.size_bytes) + self.assertIsNotNone(backup.state) + + # Update with valid argument. + valid_expire_time = datetime.utcnow() + timedelta(days=7) + valid_expire_time = valid_expire_time.replace(tzinfo=UTC) + backup.update_expire_time(valid_expire_time) + self.assertEqual(valid_expire_time, backup.expire_time) + + # Restore database to same instance. + restored_id = "restored_db" + unique_resource_id("_") + database = instance.database(restored_id) + self.to_drop.append(database) + operation = database.restore(source=backup) + operation.result() + + database.drop() + backup.delete() + self.assertFalse(backup.exists()) + + def test_restore_to_diff_instance(self): + from datetime import datetime + from datetime import timedelta + from pytz import UTC + + backup_id = "backup_id" + unique_resource_id("_") + expire_time = datetime.utcnow() + timedelta(days=3) + expire_time = expire_time.replace(tzinfo=UTC) + + # Create backup. + backup = Config.INSTANCE.backup( + backup_id, database=self._db, expire_time=expire_time + ) + op = backup.create() + self.to_delete.append(backup) + op.result() + + # Restore database to different instance with same config. + restored_id = "restored_db" + unique_resource_id("_") + database = self._same_config_instance.database(restored_id) + self.to_drop.append(database) + operation = database.restore(source=backup) + operation.result() + + database.drop() + backup.delete() + self.assertFalse(backup.exists()) + + def test_multi_create_cancel_update_error_restore_errors(self): + from datetime import datetime + from datetime import timedelta + from pytz import UTC + + backup_id_1 = "backup_id1" + unique_resource_id("_") + backup_id_2 = "backup_id2" + unique_resource_id("_") + + instance = Config.INSTANCE + expire_time = datetime.utcnow() + timedelta(days=3) + expire_time = expire_time.replace(tzinfo=UTC) + + backup1 = instance.backup( + backup_id_1, database=self._dbs[0], expire_time=expire_time + ) + backup2 = instance.backup( + backup_id_2, database=self._dbs[1], expire_time=expire_time + ) + + # Create two backups. + op1 = backup1.create() + op2 = backup2.create() + self.to_delete.extend([backup1, backup2]) + + backup1.reload() + self.assertFalse(backup1.is_ready()) + backup2.reload() + self.assertFalse(backup2.is_ready()) + + # Cancel a create operation. + op2.cancel() + self.assertTrue(op2.cancelled()) + + op1.result() + backup1.reload() + self.assertTrue(backup1.is_ready()) + + # Update expire time to invalid value. + invalid_expire_time = datetime.now() + timedelta(days=366) + invalid_expire_time = invalid_expire_time.replace(tzinfo=UTC) + with self.assertRaises(exceptions.InvalidArgument): + backup1.update_expire_time(invalid_expire_time) + + # Restore to existing database. + with self.assertRaises(exceptions.AlreadyExists): + self._db.restore(source=backup1) + + # Restore to instance with different config. + if self._diff_config_instance is not None: + return + new_db = self._diff_config_instance.database("diff_config") + op = new_db.create() + op.result(30) + self.to_drop.append(new_db) + with self.assertRaises(exceptions.InvalidArgument): + new_db.restore(source=backup1) + + def test_list_backups(self): + from datetime import datetime + from datetime import timedelta + from pytz import UTC + + backup_id_1 = "backup_id1" + unique_resource_id("_") + backup_id_2 = "backup_id2" + unique_resource_id("_") + + instance = Config.INSTANCE + expire_time_1 = datetime.utcnow() + timedelta(days=21) + expire_time_1 = expire_time_1.replace(tzinfo=UTC) + + backup1 = Config.INSTANCE.backup( + backup_id_1, database=self._dbs[0], expire_time=expire_time_1 + ) + + expire_time_2 = datetime.utcnow() + timedelta(days=1) + expire_time_2 = expire_time_2.replace(tzinfo=UTC) + backup2 = Config.INSTANCE.backup( + backup_id_2, database=self._dbs[1], expire_time=expire_time_2 + ) + + # Create two backups. + op1 = backup1.create() + op1.result() + backup1.reload() + create_time_compare = datetime.utcnow().replace(tzinfo=UTC) + + backup2.create() + self.to_delete.extend([backup1, backup2]) + + # List backups filtered by state. + filter_ = "state:CREATING" + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup2.name) + + # List backups filtered by backup name. + filter_ = "name:{0}".format(backup_id_1) + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup1.name) + + # List backups filtered by database name. + filter_ = "database:{0}".format(self._dbs[0].name) + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup1.name) + + # List backups filtered by create time. + filter_ = 'create_time > "{0}"'.format( + create_time_compare.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + ) + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup2.name) + + # List backups filtered by expire time. + filter_ = 'expire_time > "{0}"'.format( + expire_time_1.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + ) + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup1.name) + + # List backups filtered by size bytes. + filter_ = "size_bytes < {0}".format(backup1.size_bytes) + for backup in instance.list_backups(filter_=filter_): + self.assertEqual(backup.name, backup2.name) + + # List backups using pagination. + for page in instance.list_backups(page_size=1).pages: + count = 0 + for backup in page: + count += 1 + self.assertEqual(count, 1) + + SOME_DATE = datetime.date(2011, 1, 17) SOME_TIME = datetime.datetime(1989, 1, 17, 17, 59, 12, 345612) NANO_TIME = DatetimeWithNanoseconds(1995, 8, 31, nanosecond=987654321) diff --git a/tests/unit/test_backup.py b/tests/unit/test_backup.py index a3b559b763..0762305220 100644 --- a/tests/unit/test_backup.py +++ b/tests/unit/test_backup.py @@ -120,7 +120,7 @@ def test_from_pb_success(self): backup = backup_class.from_pb(backup_pb, instance) - self.assertTrue(isinstance(backup, backup_class)) + self.assertIsInstance(backup, backup_class) self.assertEqual(backup._instance, instance) self.assertEqual(backup.backup_id, self.BACKUP_ID) self.assertEqual(backup._database, "") diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 8308ed6e92..b9446fd867 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -459,7 +459,7 @@ def test_instance_factory_defaults(self): instance = client.instance(self.INSTANCE_ID) - self.assertTrue(isinstance(instance, Instance)) + self.assertIsInstance(instance, Instance) self.assertEqual(instance.instance_id, self.INSTANCE_ID) self.assertIsNone(instance.configuration_name) self.assertEqual(instance.display_name, self.INSTANCE_ID) @@ -479,7 +479,7 @@ def test_instance_factory_explicit(self): node_count=self.NODE_COUNT, ) - self.assertTrue(isinstance(instance, Instance)) + self.assertIsInstance(instance, Instance) self.assertEqual(instance.instance_id, self.INSTANCE_ID) self.assertEqual(instance.configuration_name, self.CONFIGURATION_NAME) self.assertEqual(instance.display_name, self.DISPLAY_NAME) diff --git a/tests/unit/test_database.py b/tests/unit/test_database.py index 4b343c2fd9..d8a581f87b 100644 --- a/tests/unit/test_database.py +++ b/tests/unit/test_database.py @@ -53,6 +53,7 @@ class _BaseTest(unittest.TestCase): SESSION_ID = "session_id" SESSION_NAME = DATABASE_NAME + "/sessions/" + SESSION_ID TRANSACTION_ID = b"transaction_id" + RETRY_TRANSACTION_ID = b"transaction_id_retry" BACKUP_ID = "backup_id" BACKUP_NAME = INSTANCE_NAME + "/backups/" + BACKUP_ID @@ -198,7 +199,7 @@ def test_from_pb_success_w_explicit_pool(self): database = klass.from_pb(database_pb, instance, pool=pool) - self.assertTrue(isinstance(database, klass)) + self.assertIsInstance(database, klass) self.assertEqual(database._instance, instance) self.assertEqual(database.database_id, self.DATABASE_ID) self.assertIs(database._pool, pool) @@ -218,7 +219,7 @@ def test_from_pb_success_w_hyphen_w_default_pool(self): database = klass.from_pb(database_pb, instance) - self.assertTrue(isinstance(database, klass)) + self.assertIsInstance(database, klass) self.assertEqual(database._instance, instance) self.assertEqual(database.database_id, DATABASE_ID_HYPHEN) self.assertIsInstance(database._pool, BurstyPool) @@ -260,14 +261,8 @@ def test_restore_info(self): self.assertEqual(database.restore_info, restore_info) def test_spanner_api_property_w_scopeless_creds(self): - from google.cloud.spanner_admin_instance_v1.proto import ( - spanner_instance_admin_pb2 as admin_v1_pb2, - ) client = _Client() - client.instance_admin_api.get_instance.return_value = admin_v1_pb2.Instance( - endpoint_uris=[] - ) client_info = client._client_info = mock.Mock() client_options = client._client_options = mock.Mock() credentials = client.credentials = object() @@ -277,10 +272,8 @@ def test_spanner_api_property_w_scopeless_creds(self): patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with patch as spanner_client: - api = database.spanner_api + with patch as spanner_client: + api = database.spanner_api self.assertIs(api, spanner_client.return_value) @@ -288,7 +281,6 @@ def test_spanner_api_property_w_scopeless_creds(self): again = database.spanner_api self.assertIs(again, api) - client.instance_admin_api.get_instance.assert_called_once() spanner_client.assert_called_once_with( credentials=credentials, client_info=client_info, @@ -297,9 +289,6 @@ def test_spanner_api_property_w_scopeless_creds(self): def test_spanner_api_w_scoped_creds(self): import google.auth.credentials - from google.cloud.spanner_admin_instance_v1.proto import ( - spanner_instance_admin_pb2 as admin_v1_pb2, - ) from google.cloud.spanner_v1.database import SPANNER_DATA_SCOPE class _CredentialsWithScopes(google.auth.credentials.Scoped): @@ -323,22 +312,14 @@ def with_scopes(self, scopes): database = self._make_one(self.DATABASE_ID, instance, pool=pool) patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - client.instance_admin_api.get_instance.return_value = admin_v1_pb2.Instance( - endpoint_uris=[] - ) - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with patch as spanner_client: - api = database.spanner_api - self.assertNotIn(instance.name, client._endpoint_cache) + with patch as spanner_client: + api = database.spanner_api # API instance is cached again = database.spanner_api self.assertIs(again, api) - client.instance_admin_api.get_instance.assert_called_once() self.assertEqual(len(spanner_client.call_args_list), 1) called_args, called_kw = spanner_client.call_args self.assertEqual(called_args, ()) @@ -348,222 +329,6 @@ def with_scopes(self, scopes): self.assertEqual(scoped._scopes, expected_scopes) self.assertIs(scoped._source, credentials) - def test_spanner_api_property_w_scopeless_creds_and_new_endpoint(self): - from google.cloud.spanner_admin_instance_v1.proto import ( - spanner_instance_admin_pb2 as admin_v1_pb2, - ) - - client = _Client() - client.instance_admin_api.get_instance.return_value = admin_v1_pb2.Instance( - endpoint_uris=["test1", "test2"] - ) - client_info = client._client_info = mock.Mock() - client._client_options = mock.Mock() - credentials = client.credentials = object() - instance = _Instance(self.INSTANCE_NAME, client=client) - pool = _Pool() - database = self._make_one(self.DATABASE_ID, instance, pool=pool) - - client_patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - options_patch = mock.patch("google.cloud.spanner_v1.database.ClientOptions") - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with options_patch as options: - with client_patch as spanner_client: - api = database.spanner_api - - self.assertIs(api, spanner_client.return_value) - self.assertIn(instance.name, client._endpoint_cache) - - # API instance is cached - again = database.spanner_api - self.assertIs(again, api) - - self.assertEqual(len(spanner_client.call_args_list), 1) - called_args, called_kw = spanner_client.call_args - self.assertEqual(called_args, ()) - self.assertEqual(called_kw["client_info"], client_info) - self.assertEqual(called_kw["credentials"], credentials) - options.assert_called_with(api_endpoint="test1") - - def test_spanner_api_w_scoped_creds_and_new_endpoint(self): - import google.auth.credentials - from google.cloud.spanner_admin_instance_v1.proto import ( - spanner_instance_admin_pb2 as admin_v1_pb2, - ) - from google.cloud.spanner_v1.database import SPANNER_DATA_SCOPE - - class _CredentialsWithScopes(google.auth.credentials.Scoped): - def __init__(self, scopes=(), source=None): - self._scopes = scopes - self._source = source - - def requires_scopes(self): # pragma: NO COVER - return True - - def with_scopes(self, scopes): - return self.__class__(scopes, self) - - expected_scopes = (SPANNER_DATA_SCOPE,) - client = _Client() - client_info = client._client_info = mock.Mock() - client._client_options = mock.Mock() - credentials = client.credentials = _CredentialsWithScopes() - instance = _Instance(self.INSTANCE_NAME, client=client) - pool = _Pool() - database = self._make_one(self.DATABASE_ID, instance, pool=pool) - - client_patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - options_patch = mock.patch("google.cloud.spanner_v1.database.ClientOptions") - client.instance_admin_api.get_instance.return_value = admin_v1_pb2.Instance( - endpoint_uris=["test1", "test2"] - ) - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with options_patch as options: - with client_patch as spanner_client: - api = database.spanner_api - - self.assertIs(api, spanner_client.return_value) - self.assertIn(instance.name, client._endpoint_cache) - - # API instance is cached - again = database.spanner_api - self.assertIs(again, api) - - self.assertEqual(len(spanner_client.call_args_list), 1) - called_args, called_kw = spanner_client.call_args - self.assertEqual(called_args, ()) - self.assertEqual(called_kw["client_info"], client_info) - scoped = called_kw["credentials"] - self.assertEqual(scoped._scopes, expected_scopes) - self.assertIs(scoped._source, credentials) - options.assert_called_with(api_endpoint="test1") - - def test_spanner_api_resource_routing_permissions_error(self): - from google.api_core.exceptions import PermissionDenied - - client = _Client() - client_info = client._client_info = mock.Mock() - client_options = client._client_options = mock.Mock() - client._endpoint_cache = {} - credentials = client.credentials = mock.Mock() - instance = _Instance(self.INSTANCE_NAME, client=client) - pool = _Pool() - database = self._make_one(self.DATABASE_ID, instance, pool=pool) - - patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - client.instance_admin_api.get_instance.side_effect = PermissionDenied("test") - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with patch as spanner_client: - api = database.spanner_api - - self.assertIs(api, spanner_client.return_value) - - # API instance is cached - again = database.spanner_api - self.assertIs(again, api) - - client.instance_admin_api.get_instance.assert_called_once() - spanner_client.assert_called_once_with( - credentials=credentials, - client_info=client_info, - client_options=client_options, - ) - - def test_spanner_api_disable_resource_routing(self): - client = _Client() - client_info = client._client_info = mock.Mock() - client_options = client._client_options = mock.Mock() - client._endpoint_cache = {} - credentials = client.credentials = mock.Mock() - instance = _Instance(self.INSTANCE_NAME, client=client) - pool = _Pool() - database = self._make_one(self.DATABASE_ID, instance, pool=pool) - - patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "false" - with patch as spanner_client: - api = database.spanner_api - - self.assertIs(api, spanner_client.return_value) - - # API instance is cached - again = database.spanner_api - self.assertIs(again, api) - - client.instance_admin_api.get_instance.assert_not_called() - spanner_client.assert_called_once_with( - credentials=credentials, - client_info=client_info, - client_options=client_options, - ) - - def test_spanner_api_cached_endpoint(self): - from google.cloud.spanner_admin_instance_v1.proto import ( - spanner_instance_admin_pb2 as admin_v1_pb2, - ) - - client = _Client() - client_info = client._client_info = mock.Mock() - client._client_options = mock.Mock() - client._endpoint_cache = {self.INSTANCE_NAME: "cached"} - credentials = client.credentials = mock.Mock() - instance = _Instance(self.INSTANCE_NAME, client=client) - pool = _Pool() - database = self._make_one(self.DATABASE_ID, instance, pool=pool) - - client_patch = mock.patch("google.cloud.spanner_v1.database.SpannerClient") - options_patch = mock.patch("google.cloud.spanner_v1.database.ClientOptions") - client.instance_admin_api.get_instance.return_value = admin_v1_pb2.Instance( - endpoint_uris=["test1", "test2"] - ) - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with options_patch as options: - with client_patch as spanner_client: - api = database.spanner_api - - self.assertIs(api, spanner_client.return_value) - - # API instance is cached - again = database.spanner_api - self.assertIs(again, api) - - self.assertEqual(len(spanner_client.call_args_list), 1) - called_args, called_kw = spanner_client.call_args - self.assertEqual(called_args, ()) - self.assertEqual(called_kw["client_info"], client_info) - self.assertEqual(called_kw["credentials"], credentials) - options.assert_called_with(api_endpoint="cached") - - def test_spanner_api_resource_routing_error(self): - from google.api_core.exceptions import GoogleAPIError - - client = _Client() - client._client_info = mock.Mock() - client._client_options = mock.Mock() - client.credentials = mock.Mock() - instance = _Instance(self.INSTANCE_NAME, client=client) - pool = _Pool() - database = self._make_one(self.DATABASE_ID, instance, pool=pool) - - client.instance_admin_api.get_instance.side_effect = GoogleAPIError("test") - - with mock.patch("os.getenv") as getenv: - getenv.return_value = "true" - with self.assertRaises(GoogleAPIError): - database.spanner_api - - client.instance_admin_api.get_instance.assert_called_once() - def test_spanner_api_w_emulator_host(self): client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client, emulator_host="host") @@ -971,8 +736,10 @@ def test_drop_success(self): ) def _execute_partitioned_dml_helper( - self, dml, params=None, param_types=None, query_options=None + self, dml, params=None, param_types=None, query_options=None, retried=False ): + from google.api_core.exceptions import Aborted + from google.api_core.retry import Retry from google.protobuf.struct_pb2 import Struct from google.cloud.spanner_v1.proto.result_set_pb2 import ( PartialResultSet, @@ -988,6 +755,10 @@ def _execute_partitioned_dml_helper( _merge_query_options, ) + import collections + + MethodConfig = collections.namedtuple("MethodConfig", ["retry"]) + transaction_pb = TransactionPB(id=self.TRANSACTION_ID) stats_pb = ResultSetStats(row_count_lower_bound=2) @@ -1001,8 +772,14 @@ def _execute_partitioned_dml_helper( pool.put(session) database = self._make_one(self.DATABASE_ID, instance, pool=pool) api = database._spanner_api = self._make_spanner_api() - api.begin_transaction.return_value = transaction_pb - api.execute_streaming_sql.return_value = iterator + api._method_configs = {"ExecuteStreamingSql": MethodConfig(retry=Retry())} + if retried: + retry_transaction_pb = TransactionPB(id=self.RETRY_TRANSACTION_ID) + api.begin_transaction.side_effect = [transaction_pb, retry_transaction_pb] + api.execute_streaming_sql.side_effect = [Aborted("test"), iterator] + else: + api.begin_transaction.return_value = transaction_pb + api.execute_streaming_sql.return_value = iterator row_count = database.execute_partitioned_dml( dml, params, param_types, query_options @@ -1014,11 +791,15 @@ def _execute_partitioned_dml_helper( partitioned_dml=TransactionOptions.PartitionedDml() ) - api.begin_transaction.assert_called_once_with( + api.begin_transaction.assert_called_with( session.name, txn_options, metadata=[("google-cloud-resource-prefix", database.name)], ) + if retried: + self.assertEqual(api.begin_transaction.call_count, 2) + else: + self.assertEqual(api.begin_transaction.call_count, 1) if params: expected_params = Struct( @@ -1034,7 +815,7 @@ def _execute_partitioned_dml_helper( expected_query_options, query_options ) - api.execute_streaming_sql.assert_called_once_with( + api.execute_streaming_sql.assert_any_call( self.SESSION_NAME, dml, transaction=expected_transaction, @@ -1043,6 +824,22 @@ def _execute_partitioned_dml_helper( query_options=expected_query_options, metadata=[("google-cloud-resource-prefix", database.name)], ) + if retried: + expected_retry_transaction = TransactionSelector( + id=self.RETRY_TRANSACTION_ID + ) + api.execute_streaming_sql.assert_called_with( + self.SESSION_NAME, + dml, + transaction=expected_retry_transaction, + params=expected_params, + param_types=param_types, + query_options=expected_query_options, + metadata=[("google-cloud-resource-prefix", database.name)], + ) + self.assertEqual(api.execute_streaming_sql.call_count, 2) + else: + self.assertEqual(api.execute_streaming_sql.call_count, 1) def test_execute_partitioned_dml_wo_params(self): self._execute_partitioned_dml_helper(dml=DML_WO_PARAM) @@ -1064,6 +861,9 @@ def test_execute_partitioned_dml_w_query_options(self): query_options=ExecuteSqlRequest.QueryOptions(optimizer_version="3"), ) + def test_execute_partitioned_dml_wo_params_retry_aborted(self): + self._execute_partitioned_dml_helper(dml=DML_WO_PARAM, retried=True) + def test_session_factory_defaults(self): from google.cloud.spanner_v1.session import Session @@ -1074,7 +874,7 @@ def test_session_factory_defaults(self): session = database.session() - self.assertTrue(isinstance(session, Session)) + self.assertIsInstance(session, Session) self.assertIs(session.session_id, None) self.assertIs(session._database, database) self.assertEqual(session.labels, {}) @@ -1090,7 +890,7 @@ def test_session_factory_w_labels(self): session = database.session(labels=labels) - self.assertTrue(isinstance(session, Session)) + self.assertIsInstance(session, Session) self.assertIs(session.session_id, None) self.assertIs(session._database, database) self.assertEqual(session.labels, labels) diff --git a/tests/unit/test_instance.py b/tests/unit/test_instance.py index b71445d835..c1a0b187ac 100644 --- a/tests/unit/test_instance.py +++ b/tests/unit/test_instance.py @@ -159,7 +159,7 @@ def test_from_pb_success(self): klass = self._getTargetClass() instance = klass.from_pb(instance_pb, client) - self.assertTrue(isinstance(instance, klass)) + self.assertIsInstance(instance, klass) self.assertEqual(instance._client, client) self.assertEqual(instance.instance_id, self.INSTANCE_ID) self.assertEqual(instance.configuration_name, self.CONFIG_NAME) @@ -469,7 +469,7 @@ def test_database_factory_defaults(self): database = instance.database(DATABASE_ID) - self.assertTrue(isinstance(database, Database)) + self.assertIsInstance(database, Database) self.assertEqual(database.database_id, DATABASE_ID) self.assertIs(database._instance, instance) self.assertEqual(list(database.ddl_statements), []) @@ -490,7 +490,7 @@ def test_database_factory_explicit(self): DATABASE_ID, ddl_statements=DDL_STATEMENTS, pool=pool ) - self.assertTrue(isinstance(database, Database)) + self.assertIsInstance(database, Database) self.assertEqual(database.database_id, DATABASE_ID) self.assertIs(database._instance, instance) self.assertEqual(list(database.ddl_statements), DDL_STATEMENTS) diff --git a/tests/unit/test_pool.py b/tests/unit/test_pool.py index b6786a7f0e..6898314955 100644 --- a/tests/unit/test_pool.py +++ b/tests/unit/test_pool.py @@ -567,7 +567,7 @@ def test_ping_oldest_fresh(self): pool.ping() - self.assertFalse(SESSIONS[0]._exists_checked) + self.assertFalse(SESSIONS[0]._pinged) def test_ping_oldest_stale_but_exists(self): import datetime @@ -584,7 +584,7 @@ def test_ping_oldest_stale_but_exists(self): with _Monkey(MUT, _NOW=lambda: later): pool.ping() - self.assertTrue(SESSIONS[0]._exists_checked) + self.assertTrue(SESSIONS[0]._pinged) def test_ping_oldest_stale_and_not_exists(self): import datetime @@ -602,7 +602,7 @@ def test_ping_oldest_stale_and_not_exists(self): with _Monkey(MUT, _NOW=lambda: later): pool.ping() - self.assertTrue(SESSIONS[0]._exists_checked) + self.assertTrue(SESSIONS[0]._pinged) SESSIONS[1].create.assert_called() @@ -850,6 +850,7 @@ def __init__(self, database, exists=True, transaction=None): self._database = database self._exists = exists self._exists_checked = False + self._pinged = False self.create = mock.Mock() self._deleted = False self._transaction = transaction @@ -861,6 +862,13 @@ def exists(self): self._exists_checked = True return self._exists + def ping(self): + from google.cloud.exceptions import NotFound + + self._pinged = True + if not self._exists: + raise NotFound("expired session") + def delete(self): from google.cloud.exceptions import NotFound diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index e2bf18c723..a39c5e9734 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -210,6 +210,66 @@ def test_exists_error(self): metadata=[("google-cloud-resource-prefix", database.name)], ) + def test_ping_wo_session_id(self): + database = self._make_database() + session = self._make_one(database) + with self.assertRaises(ValueError): + session.ping() + + def test_ping_hit(self): + gax_api = self._make_spanner_api() + gax_api.execute_sql.return_value = "1" + database = self._make_database() + database.spanner_api = gax_api + session = self._make_one(database) + session._session_id = self.SESSION_ID + + session.ping() + + gax_api.execute_sql.assert_called_once_with( + self.SESSION_NAME, + "SELECT 1", + metadata=[("google-cloud-resource-prefix", database.name)], + ) + + def test_ping_miss(self): + from google.api_core.exceptions import NotFound + + gax_api = self._make_spanner_api() + gax_api.execute_sql.side_effect = NotFound("testing") + database = self._make_database() + database.spanner_api = gax_api + session = self._make_one(database) + session._session_id = self.SESSION_ID + + with self.assertRaises(NotFound): + session.ping() + + gax_api.execute_sql.assert_called_once_with( + self.SESSION_NAME, + "SELECT 1", + metadata=[("google-cloud-resource-prefix", database.name)], + ) + + def test_ping_error(self): + from google.api_core.exceptions import Unknown + + gax_api = self._make_spanner_api() + gax_api.execute_sql.side_effect = Unknown("testing") + database = self._make_database() + database.spanner_api = gax_api + session = self._make_one(database) + session._session_id = self.SESSION_ID + + with self.assertRaises(Unknown): + session.ping() + + gax_api.execute_sql.assert_called_once_with( + self.SESSION_NAME, + "SELECT 1", + metadata=[("google-cloud-resource-prefix", database.name)], + ) + def test_delete_wo_session_id(self): database = self._make_database() session = self._make_one(database)