From ff6ef1bd07fa68307b7c82c910416d770e7b3416 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Tue, 3 Aug 2021 11:47:13 -0600 Subject: [PATCH 001/112] fix: strip trailing _ from field mask paths (#228) --- google/api_core/protobuf_helpers.py | 7 +++++ noxfile.py | 5 ++-- tests/unit/test_protobuf_helpers.py | 44 +++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/google/api_core/protobuf_helpers.py b/google/api_core/protobuf_helpers.py index 365ef25c..8aff79aa 100644 --- a/google/api_core/protobuf_helpers.py +++ b/google/api_core/protobuf_helpers.py @@ -357,6 +357,13 @@ def _field_mask_helper(original, modified, current=""): def _get_path(current, name): + # gapic-generator-python appends underscores to field names + # that collide with python keywords. + # `_` is stripped away as it is not possible to + # natively define a field with a trailing underscore in protobuf. + # APIs will reject field masks if fields have trailing underscores. + # See https://github.com/googleapis/python-api-core/issues/227 + name = name.rstrip("_") if not current: return name return "%s.%s" % (current, name) diff --git a/noxfile.py b/noxfile.py index 25609920..2f11137d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -98,9 +98,10 @@ def default(session): ] pytest_args.extend(session.posargs) - # Inject AsyncIO content, if version >= 3.6. + # Inject AsyncIO content and proto-plus, if version >= 3.6. + # proto-plus is needed for a field mask test in test_protobuf_helpers.py if _greater_or_equal_than_36(session.python): - session.install("asyncmock", "pytest-asyncio") + session.install("asyncmock", "pytest-asyncio", "proto-plus") pytest_args.append("--cov=tests.asyncio") pytest_args.append(os.path.join("tests", "asyncio")) diff --git a/tests/unit/test_protobuf_helpers.py b/tests/unit/test_protobuf_helpers.py index db972383..3df45df5 100644 --- a/tests/unit/test_protobuf_helpers.py +++ b/tests/unit/test_protobuf_helpers.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys + import pytest from google.api import http_pb2 @@ -472,3 +474,45 @@ def test_field_mask_different_level_diffs(): "alpha", "red", ] + + +@pytest.mark.skipif( + sys.version_info.major == 2, + reason="Field names with trailing underscores can only be created" + "through proto-plus, which is Python 3 only.", +) +def test_field_mask_ignore_trailing_underscore(): + import proto + + class Foo(proto.Message): + type_ = proto.Field(proto.STRING, number=1) + input_config = proto.Field(proto.STRING, number=2) + + modified = Foo(type_="bar", input_config="baz") + + assert sorted(protobuf_helpers.field_mask(None, Foo.pb(modified)).paths) == [ + "input_config", + "type", + ] + + +@pytest.mark.skipif( + sys.version_info.major == 2, + reason="Field names with trailing underscores can only be created" + "through proto-plus, which is Python 3 only.", +) +def test_field_mask_ignore_trailing_underscore_with_nesting(): + import proto + + class Bar(proto.Message): + class Baz(proto.Message): + input_config = proto.Field(proto.STRING, number=1) + + type_ = proto.Field(Baz, number=1) + + modified = Bar() + modified.type_.input_config = "foo" + + assert sorted(protobuf_helpers.field_mask(None, Bar.pb(modified)).paths) == [ + "type.input_config", + ] From a30f004e74f709d46e905dd819c71f43354e9ac9 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 3 Aug 2021 13:59:25 -0400 Subject: [PATCH 002/112] fix!: drop support for Python 2.7 / 3.5 (#212) Drop 'six' module Drop 'u"' prefixes for text Remove other Python 2.7 workarounds Drop use of 'pytz' Dxpand range to allow 'google-auth' 2.x versions Remove 'general_helpers.wraps': except for a backward-compatibility import, 'functools.wraps' does everything wee need on Python >= 3.6. Remove 'packaging' dependency Release-As: 2.0.0b1 Closes #74. Closes #215. --- CONTRIBUTING.rst | 34 ++------------- README.rst | 15 ++++--- docs/auth.rst | 19 -------- google/api_core/bidi.py | 10 ++--- google/api_core/client_info.py | 2 +- google/api_core/client_options.py | 2 +- google/api_core/datetime_helpers.py | 20 +++++---- google/api_core/exceptions.py | 47 +++++++++----------- google/api_core/future/base.py | 5 +-- google/api_core/gapic_v1/__init__.py | 20 ++++----- google/api_core/gapic_v1/client_info.py | 2 +- google/api_core/gapic_v1/config.py | 11 ++--- google/api_core/gapic_v1/method.py | 19 ++++---- google/api_core/gapic_v1/method_async.py | 6 ++- google/api_core/gapic_v1/routing_header.py | 7 +-- google/api_core/general_helpers.py | 21 +-------- google/api_core/grpc_helpers.py | 22 ++++------ google/api_core/iam.py | 8 +--- google/api_core/operations_v1/__init__.py | 8 +--- google/api_core/page_iterator.py | 17 ++------ google/api_core/path_template.py | 6 +-- google/api_core/protobuf_helpers.py | 20 ++++----- google/api_core/retry.py | 16 +++---- google/api_core/timeout.py | 10 ++--- google/api_core/version.py | 2 +- noxfile.py | 5 +-- owlbot.py | 11 ++++- setup.py | 10 +---- testing/constraints-2.7.txt | 1 - testing/constraints-3.6.txt | 1 - tests/unit/test_bidi.py | 10 +---- tests/unit/test_datetime_helpers.py | 51 ++++++++++++---------- tests/unit/test_exceptions.py | 32 +++++++------- tests/unit/test_general_helpers.py | 41 ----------------- tests/unit/test_page_iterator.py | 29 ++++++------ 35 files changed, 191 insertions(+), 349 deletions(-) delete mode 100644 testing/constraints-2.7.txt delete mode 100644 tests/unit/test_general_helpers.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c7860adb..358404f2 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,4 +1,3 @@ -.. Generated by synthtool. DO NOT EDIT! ############ Contributing ############ @@ -22,7 +21,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 2.7, 3.6, 3.7, 3.8 and 3.9 on both UNIX and Windows. + 3.6, 3.7, 3.8 and 3.9 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -77,8 +76,8 @@ We use `nox `__ to instrument our tests. .. note:: - The unit tests and system tests are described in the - ``noxfile.py`` files in each directory. + The unit tests tests are described in the ``noxfile.py`` files + in each directory. .. nox: https://pypi.org/project/nox/ @@ -133,29 +132,6 @@ Exceptions to PEP8: "Function-Under-Test"), which is PEP8-incompliant, but more readable. Some also use a local variable, ``MUT`` (short for "Module-Under-Test"). -******************** -Running System Tests -******************** - -- To run system tests, you can execute:: - - # Run all system tests - $ nox -s system - - # Run a single system test - $ nox -s system-3.8 -- -k - - - .. note:: - - System tests are only configured to run under Python 2.7 and 3.8. - For expediency, we do not run them in older versions of Python 3. - - This alone will not run the tests. You'll need to change some local - auth settings and change some configuration in your project to - run all the tests. - -- System tests will be run against an actual project. You should use local credentials from gcloud when possible. See `Best practices for application authentication `__. Some tests require a service account. For those tests see `Authenticating as a service account `__. ************* Test Coverage @@ -221,13 +197,11 @@ Supported Python Versions We support: -- `Python 2.7`_ - `Python 3.6`_ - `Python 3.7`_ - `Python 3.8`_ - `Python 3.9`_ -.. _Python 2.7: https://docs.python.org/2.7/ .. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ @@ -239,7 +213,7 @@ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-api-core/blob/master/noxfile.py -We also explicitly decided to support Python 3 beginning with version 2.7. +We also explicitly decided to support Python 3 beginning with version 3.6. Reasons for this include: - Encouraging use of newest versions of Python 3 diff --git a/README.rst b/README.rst index 244043ea..d94f3e88 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ Core Library for Google Client Libraries ======================================== -|pypi| |versions| +|pypi| |versions| This library is not meant to stand-alone. Instead it defines common helpers used by all Google API clients. For more information, see the @@ -16,8 +16,13 @@ common helpers used by all Google API clients. For more information, see the Supported Python Versions ------------------------- -Python >= 3.5 +Python >= 3.6 -Deprecated Python Versions --------------------------- -Python == 2.7. Python 2.7 support will be removed on January 1, 2020. + +Unsupported Python Versions +--------------------------- + +Python == 2.7, Python == 3.5. + +The last version of this library compatible with Python 2.7 and 3.5 is +`google-api_core==1.31.1`. diff --git a/docs/auth.rst b/docs/auth.rst index faf0228f..a9b296d4 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -103,25 +103,6 @@ After creation, you can pass it directly to a :class:`Client ` -just for Google App Engine: - -.. code:: python - - from google.auth import app_engine - credentials = app_engine.Credentials() - Google Compute Engine Environment --------------------------------- diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index be52d97d..56a021a9 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -17,11 +17,10 @@ import collections import datetime import logging +import queue as queue_module import threading import time -from six.moves import queue - from google.api_core import exceptions _LOGGER = logging.getLogger(__name__) @@ -71,7 +70,7 @@ class _RequestQueueGenerator(object): CPU consumed by spinning is pretty minuscule. Args: - queue (queue.Queue): The request queue. + queue (queue_module.Queue): The request queue. period (float): The number of seconds to wait for items from the queue before checking if the RPC is cancelled. In practice, this determines the maximum amount of time the request consumption @@ -108,7 +107,7 @@ def __iter__(self): while True: try: item = self._queue.get(timeout=self._period) - except queue.Empty: + except queue_module.Empty: if not self._is_active(): _LOGGER.debug( "Empty queue and inactive call, exiting request " "generator." @@ -247,7 +246,7 @@ def __init__(self, start_rpc, initial_request=None, metadata=None): self._start_rpc = start_rpc self._initial_request = initial_request self._rpc_metadata = metadata - self._request_queue = queue.Queue() + self._request_queue = queue_module.Queue() self._request_generator = None self._is_active = False self._callbacks = [] @@ -645,6 +644,7 @@ def _thread_main(self, ready): # Keeping the lock throughout avoids that. # In the future, we could use `Condition.wait_for` if we drop # Python 2.7. + # See: https://github.com/googleapis/python-api-core/issues/211 with self._wake: while self._paused: _LOGGER.debug("paused, waiting for waking.") diff --git a/google/api_core/client_info.py b/google/api_core/client_info.py index adca5f32..d7f4367a 100644 --- a/google/api_core/client_info.py +++ b/google/api_core/client_info.py @@ -42,7 +42,7 @@ class ClientInfo(object): Args: python_version (str): The Python interpreter version, for example, - ``'2.7.13'``. + ``'3.9.6'``. grpc_version (Optional[str]): The gRPC library version. api_core_version (str): The google-api-core library version. gapic_version (Optional[str]): The sversion of gapic-generated client diff --git a/google/api_core/client_options.py b/google/api_core/client_options.py index 57000e95..be5523df 100644 --- a/google/api_core/client_options.py +++ b/google/api_core/client_options.py @@ -101,7 +101,7 @@ def from_dict(options): """Construct a client options object from a mapping object. Args: - options (six.moves.collections_abc.Mapping): A mapping object with client options. + options (collections.abc.Mapping): A mapping object with client options. See the docstring for ClientOptions for details on valid arguments. """ diff --git a/google/api_core/datetime_helpers.py b/google/api_core/datetime_helpers.py index e52fb1dd..78268efc 100644 --- a/google/api_core/datetime_helpers.py +++ b/google/api_core/datetime_helpers.py @@ -18,12 +18,10 @@ import datetime import re -import pytz - from google.protobuf import timestamp_pb2 -_UTC_EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc) +_UTC_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) _RFC3339_MICROS = "%Y-%m-%dT%H:%M:%S.%fZ" _RFC3339_NO_FRACTION = "%Y-%m-%dT%H:%M:%S" # datetime.strptime cannot handle nanosecond precision: parse w/ regex @@ -83,9 +81,9 @@ def to_microseconds(value): int: Microseconds since the unix epoch. """ if not value.tzinfo: - value = value.replace(tzinfo=pytz.utc) + value = value.replace(tzinfo=datetime.timezone.utc) # Regardless of what timezone is on the value, convert it to UTC. - value = value.astimezone(pytz.utc) + value = value.astimezone(datetime.timezone.utc) # Convert the datetime to a microsecond timestamp. return int(calendar.timegm(value.timetuple()) * 1e6) + value.microsecond @@ -156,7 +154,7 @@ def from_rfc3339(value): nanos = int(fraction) * (10 ** scale) micros = nanos // 1000 - return bare_seconds.replace(microsecond=micros, tzinfo=pytz.utc) + return bare_seconds.replace(microsecond=micros, tzinfo=datetime.timezone.utc) from_rfc3339_nanos = from_rfc3339 # from_rfc3339_nanos method was deprecated. @@ -256,7 +254,7 @@ def from_rfc3339(cls, stamp): bare.minute, bare.second, nanosecond=nanos, - tzinfo=pytz.UTC, + tzinfo=datetime.timezone.utc, ) def timestamp_pb(self): @@ -265,7 +263,11 @@ def timestamp_pb(self): Returns: (:class:`~google.protobuf.timestamp_pb2.Timestamp`): Timestamp message """ - inst = self if self.tzinfo is not None else self.replace(tzinfo=pytz.UTC) + inst = ( + self + if self.tzinfo is not None + else self.replace(tzinfo=datetime.timezone.utc) + ) delta = inst - _UTC_EPOCH seconds = int(delta.total_seconds()) nanos = self._nanosecond or self.microsecond * 1000 @@ -292,5 +294,5 @@ def from_timestamp_pb(cls, stamp): bare.minute, bare.second, nanosecond=stamp.nanos, - tzinfo=pytz.UTC, + tzinfo=datetime.timezone.utc, ) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 412fc2ee..13be917e 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -21,8 +21,7 @@ from __future__ import absolute_import from __future__ import unicode_literals -import six -from six.moves import http_client +import http.client try: import grpc @@ -56,7 +55,6 @@ class DuplicateCredentialArgs(GoogleAPIError): pass -@six.python_2_unicode_compatible class RetryError(GoogleAPIError): """Raised when a function has exhausted all of its available retries. @@ -92,9 +90,7 @@ def __new__(mcs, name, bases, class_dict): return cls -@six.python_2_unicode_compatible -@six.add_metaclass(_GoogleAPICallErrorMeta) -class GoogleAPICallError(GoogleAPIError): +class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): """Base class for exceptions raised by calling API methods. Args: @@ -153,25 +149,25 @@ class Redirection(GoogleAPICallError): class MovedPermanently(Redirection): """Exception mapping a ``301 Moved Permanently`` response.""" - code = http_client.MOVED_PERMANENTLY + code = http.client.MOVED_PERMANENTLY class NotModified(Redirection): """Exception mapping a ``304 Not Modified`` response.""" - code = http_client.NOT_MODIFIED + code = http.client.NOT_MODIFIED class TemporaryRedirect(Redirection): """Exception mapping a ``307 Temporary Redirect`` response.""" - code = http_client.TEMPORARY_REDIRECT + code = http.client.TEMPORARY_REDIRECT class ResumeIncomplete(Redirection): """Exception mapping a ``308 Resume Incomplete`` response. - .. note:: :attr:`http_client.PERMANENT_REDIRECT` is ``308``, but Google + .. note:: :attr:`http.client.PERMANENT_REDIRECT` is ``308``, but Google APIs differ in their use of this status code. """ @@ -185,7 +181,7 @@ class ClientError(GoogleAPICallError): class BadRequest(ClientError): """Exception mapping a ``400 Bad Request`` response.""" - code = http_client.BAD_REQUEST + code = http.client.BAD_REQUEST class InvalidArgument(BadRequest): @@ -210,7 +206,7 @@ class OutOfRange(BadRequest): class Unauthorized(ClientError): """Exception mapping a ``401 Unauthorized`` response.""" - code = http_client.UNAUTHORIZED + code = http.client.UNAUTHORIZED class Unauthenticated(Unauthorized): @@ -222,7 +218,7 @@ class Unauthenticated(Unauthorized): class Forbidden(ClientError): """Exception mapping a ``403 Forbidden`` response.""" - code = http_client.FORBIDDEN + code = http.client.FORBIDDEN class PermissionDenied(Forbidden): @@ -235,20 +231,20 @@ class NotFound(ClientError): """Exception mapping a ``404 Not Found`` response or a :attr:`grpc.StatusCode.NOT_FOUND` error.""" - code = http_client.NOT_FOUND + code = http.client.NOT_FOUND grpc_status_code = grpc.StatusCode.NOT_FOUND if grpc is not None else None class MethodNotAllowed(ClientError): """Exception mapping a ``405 Method Not Allowed`` response.""" - code = http_client.METHOD_NOT_ALLOWED + code = http.client.METHOD_NOT_ALLOWED class Conflict(ClientError): """Exception mapping a ``409 Conflict`` response.""" - code = http_client.CONFLICT + code = http.client.CONFLICT class AlreadyExists(Conflict): @@ -266,26 +262,25 @@ class Aborted(Conflict): class LengthRequired(ClientError): """Exception mapping a ``411 Length Required`` response.""" - code = http_client.LENGTH_REQUIRED + code = http.client.LENGTH_REQUIRED class PreconditionFailed(ClientError): """Exception mapping a ``412 Precondition Failed`` response.""" - code = http_client.PRECONDITION_FAILED + code = http.client.PRECONDITION_FAILED class RequestRangeNotSatisfiable(ClientError): """Exception mapping a ``416 Request Range Not Satisfiable`` response.""" - code = http_client.REQUESTED_RANGE_NOT_SATISFIABLE + code = http.client.REQUESTED_RANGE_NOT_SATISFIABLE class TooManyRequests(ClientError): """Exception mapping a ``429 Too Many Requests`` response.""" - # http_client does not define a constant for this in Python 2. - code = 429 + code = http.client.TOO_MANY_REQUESTS class ResourceExhausted(TooManyRequests): @@ -312,7 +307,7 @@ class InternalServerError(ServerError): """Exception mapping a ``500 Internal Server Error`` response. or a :attr:`grpc.StatusCode.INTERNAL` error.""" - code = http_client.INTERNAL_SERVER_ERROR + code = http.client.INTERNAL_SERVER_ERROR grpc_status_code = grpc.StatusCode.INTERNAL if grpc is not None else None @@ -332,28 +327,28 @@ class MethodNotImplemented(ServerError): """Exception mapping a ``501 Not Implemented`` response or a :attr:`grpc.StatusCode.UNIMPLEMENTED` error.""" - code = http_client.NOT_IMPLEMENTED + code = http.client.NOT_IMPLEMENTED grpc_status_code = grpc.StatusCode.UNIMPLEMENTED if grpc is not None else None class BadGateway(ServerError): """Exception mapping a ``502 Bad Gateway`` response.""" - code = http_client.BAD_GATEWAY + code = http.client.BAD_GATEWAY class ServiceUnavailable(ServerError): """Exception mapping a ``503 Service Unavailable`` response or a :attr:`grpc.StatusCode.UNAVAILABLE` error.""" - code = http_client.SERVICE_UNAVAILABLE + code = http.client.SERVICE_UNAVAILABLE grpc_status_code = grpc.StatusCode.UNAVAILABLE if grpc is not None else None class GatewayTimeout(ServerError): """Exception mapping a ``504 Gateway Timeout`` response.""" - code = http_client.GATEWAY_TIMEOUT + code = http.client.GATEWAY_TIMEOUT class DeadlineExceeded(GatewayTimeout): diff --git a/google/api_core/future/base.py b/google/api_core/future/base.py index e7888ca3..f3005860 100644 --- a/google/api_core/future/base.py +++ b/google/api_core/future/base.py @@ -16,11 +16,8 @@ import abc -import six - -@six.add_metaclass(abc.ABCMeta) -class Future(object): +class Future(object, metaclass=abc.ABCMeta): # pylint: disable=missing-docstring # We inherit the interfaces here from concurrent.futures. diff --git a/google/api_core/gapic_v1/__init__.py b/google/api_core/gapic_v1/__init__.py index 6632047a..e5b7ad35 100644 --- a/google/api_core/gapic_v1/__init__.py +++ b/google/api_core/gapic_v1/__init__.py @@ -12,18 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from google.api_core.gapic_v1 import client_info from google.api_core.gapic_v1 import config +from google.api_core.gapic_v1 import config_async from google.api_core.gapic_v1 import method +from google.api_core.gapic_v1 import method_async from google.api_core.gapic_v1 import routing_header -__all__ = ["client_info", "config", "method", "routing_header"] - -if sys.version_info >= (3, 6): - from google.api_core.gapic_v1 import config_async # noqa: F401 - from google.api_core.gapic_v1 import method_async # noqa: F401 - - __all__.append("config_async") - __all__.append("method_async") +__all__ = [ + "client_info", + "config", + "config_async", + "method", + "method_async", + "routing_header", +] diff --git a/google/api_core/gapic_v1/client_info.py b/google/api_core/gapic_v1/client_info.py index bdc2ce44..fab0f542 100644 --- a/google/api_core/gapic_v1/client_info.py +++ b/google/api_core/gapic_v1/client_info.py @@ -33,7 +33,7 @@ class ClientInfo(client_info.ClientInfo): Args: python_version (str): The Python interpreter version, for example, - ``'2.7.13'``. + ``'3.9.6'``. grpc_version (Optional[str]): The gRPC library version. api_core_version (str): The google-api-core library version. gapic_version (Optional[str]): The sversion of gapic-generated client diff --git a/google/api_core/gapic_v1/config.py b/google/api_core/gapic_v1/config.py index 29e8645b..9c722871 100644 --- a/google/api_core/gapic_v1/config.py +++ b/google/api_core/gapic_v1/config.py @@ -21,7 +21,6 @@ import collections import grpc -import six from google.api_core import exceptions from google.api_core import retry @@ -130,24 +129,20 @@ def parse_method_configs(interface_config, retry_impl=retry.Retry): # Grab all the retry codes retry_codes_map = { name: retry_codes - for name, retry_codes in six.iteritems(interface_config.get("retry_codes", {})) + for name, retry_codes in interface_config.get("retry_codes", {}).items() } # Grab all of the retry params retry_params_map = { name: retry_params - for name, retry_params in six.iteritems( - interface_config.get("retry_params", {}) - ) + for name, retry_params in interface_config.get("retry_params", {}).items() } # Iterate through all the API methods and create a flat MethodConfig # instance for each one. method_configs = {} - for method_name, method_params in six.iteritems( - interface_config.get("methods", {}) - ): + for method_name, method_params in interface_config.get("methods", {}).items(): retry_params_name = method_params.get("retry_params_name") if retry_params_name is not None: diff --git a/google/api_core/gapic_v1/method.py b/google/api_core/gapic_v1/method.py index 8bf82569..79722c07 100644 --- a/google/api_core/gapic_v1/method.py +++ b/google/api_core/gapic_v1/method.py @@ -18,7 +18,8 @@ pagination, and long-running operations to gRPC methods. """ -from google.api_core import general_helpers +import functools + from google.api_core import grpc_helpers from google.api_core import timeout from google.api_core.gapic_v1 import client_info @@ -110,26 +111,22 @@ def __init__(self, target, retry, timeout, metadata=None): self._timeout = timeout self._metadata = metadata - def __call__(self, *args, **kwargs): + def __call__(self, *args, timeout=DEFAULT, retry=DEFAULT, **kwargs): """Invoke the low-level RPC with retry, timeout, and metadata.""" - # Note: Due to Python 2 lacking keyword-only arguments we use kwargs to - # extract the retry and timeout params. - timeout_ = _determine_timeout( + timeout = _determine_timeout( self._timeout, - kwargs.pop("timeout", self._timeout), + timeout, # Use only the invocation-specified retry only for this, as we only # want to adjust the timeout deadline if the *user* specified # a different retry. - kwargs.get("retry", None), + retry, ) - retry = kwargs.pop("retry", self._retry) - if retry is DEFAULT: retry = self._retry # Apply all applicable decorators. - wrapped_func = _apply_decorators(self._target, [retry, timeout_]) + wrapped_func = _apply_decorators(self._target, [retry, timeout]) # Add the user agent metadata to the call. if self._metadata is not None: @@ -237,7 +234,7 @@ def get_topic(name, timeout=None): else: user_agent_metadata = None - return general_helpers.wraps(func)( + return functools.wraps(func)( _GapicCallable( func, default_retry, default_timeout, metadata=user_agent_metadata ) diff --git a/google/api_core/gapic_v1/method_async.py b/google/api_core/gapic_v1/method_async.py index 76e57577..84c99aa2 100644 --- a/google/api_core/gapic_v1/method_async.py +++ b/google/api_core/gapic_v1/method_async.py @@ -17,7 +17,9 @@ pagination, and long-running operations to gRPC methods. """ -from google.api_core import general_helpers, grpc_helpers_async +import functools + +from google.api_core import grpc_helpers_async from google.api_core.gapic_v1 import client_info from google.api_core.gapic_v1.method import _GapicCallable from google.api_core.gapic_v1.method import DEFAULT # noqa: F401 @@ -41,6 +43,6 @@ def wrap_method( metadata = [client_info.to_grpc_metadata()] if client_info is not None else None - return general_helpers.wraps(func)( + return functools.wraps(func)( _GapicCallable(func, default_retry, default_timeout, metadata=metadata) ) diff --git a/google/api_core/gapic_v1/routing_header.py b/google/api_core/gapic_v1/routing_header.py index 3fb12a6f..a7bcb5a8 100644 --- a/google/api_core/gapic_v1/routing_header.py +++ b/google/api_core/gapic_v1/routing_header.py @@ -20,9 +20,7 @@ Generally, these headers are specified as gRPC metadata. """ -import sys - -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode ROUTING_METADATA_KEY = "x-goog-request-params" @@ -37,9 +35,6 @@ def to_routing_header(params): Returns: str: The routing header string. """ - if sys.version_info[0] < 3: - # Python 2 does not have the "safe" parameter for urlencode. - return urlencode(params).replace("%2F", "/") return urlencode( params, # Per Google API policy (go/api-url-encoding), / is not encoded. diff --git a/google/api_core/general_helpers.py b/google/api_core/general_helpers.py index d2d0c440..fba78026 100644 --- a/google/api_core/general_helpers.py +++ b/google/api_core/general_helpers.py @@ -12,22 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Helpers for general Python functionality.""" - -import functools - -import six - - -# functools.partial objects lack several attributes present on real function -# objects. In Python 2 wraps fails on this so use a restricted set instead. -_PARTIAL_VALID_ASSIGNMENTS = ("__doc__",) - - -def wraps(wrapped): - """A functools.wraps helper that handles partial objects on Python 2.""" - # https://github.com/google/pytype/issues/322 - if isinstance(wrapped, functools.partial): # pytype: disable=wrong-arg-types - return six.wraps(wrapped, assigned=_PARTIAL_VALID_ASSIGNMENTS) - else: - return six.wraps(wrapped) +# This import for backward compatibiltiy only. +from functools import wraps # noqa: F401 pragma: NO COVER diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index 62d9e533..594df987 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -15,13 +15,12 @@ """Helpers for :mod:`grpc`.""" import collections +import functools import grpc import pkg_resources -import six from google.api_core import exceptions -from google.api_core import general_helpers import google.auth import google.auth.credentials import google.auth.transport.grpc @@ -61,12 +60,12 @@ def _wrap_unary_errors(callable_): """Map errors for Unary-Unary and Stream-Unary gRPC callables.""" _patch_callable_name(callable_) - @six.wraps(callable_) + @functools.wraps(callable_) def error_remapped_callable(*args, **kwargs): try: return callable_(*args, **kwargs) except grpc.RpcError as exc: - six.raise_from(exceptions.from_grpc_error(exc), exc) + raise exceptions.from_grpc_error(exc) from exc return error_remapped_callable @@ -80,7 +79,7 @@ def __init__(self, wrapped, prefetch_first_result=True): # to retrieve the first result, in order to fail, in order to trigger a retry. try: if prefetch_first_result: - self._stored_first_result = six.next(self._wrapped) + self._stored_first_result = next(self._wrapped) except TypeError: # It is possible the wrapped method isn't an iterable (a grpc.Call # for instance). If this happens don't store the first result. @@ -93,7 +92,7 @@ def __iter__(self): """This iterator is also an iterable that returns itself.""" return self - def next(self): + def __next__(self): """Get the next response from the stream. Returns: @@ -104,13 +103,10 @@ def next(self): result = self._stored_first_result del self._stored_first_result return result - return six.next(self._wrapped) + return next(self._wrapped) except grpc.RpcError as exc: # If the stream has already returned data, we cannot recover here. - six.raise_from(exceptions.from_grpc_error(exc), exc) - - # Alias needed for Python 2/3 support. - __next__ = next + raise exceptions.from_grpc_error(exc) from exc # grpc.Call & grpc.RpcContext interface @@ -148,7 +144,7 @@ def _wrap_stream_errors(callable_): """ _patch_callable_name(callable_) - @general_helpers.wraps(callable_) + @functools.wraps(callable_) def error_remapped_callable(*args, **kwargs): try: result = callable_(*args, **kwargs) @@ -161,7 +157,7 @@ def error_remapped_callable(*args, **kwargs): result, prefetch_first_result=prefetch_first ) except grpc.RpcError as exc: - six.raise_from(exceptions.from_grpc_error(exc), exc) + raise exceptions.from_grpc_error(exc) from exc return error_remapped_callable diff --git a/google/api_core/iam.py b/google/api_core/iam.py index 59e53874..4437c701 100644 --- a/google/api_core/iam.py +++ b/google/api_core/iam.py @@ -52,14 +52,10 @@ """ import collections +import collections.abc import operator import warnings -try: - from collections import abc as collections_abc -except ImportError: # Python 2.7 - import collections as collections_abc - # Generic IAM roles OWNER_ROLE = "roles/owner" @@ -84,7 +80,7 @@ class InvalidOperationException(Exception): pass -class Policy(collections_abc.MutableMapping): +class Policy(collections.abc.MutableMapping): """IAM Policy Args: diff --git a/google/api_core/operations_v1/__init__.py b/google/api_core/operations_v1/__init__.py index bc9befcb..d7f963e2 100644 --- a/google/api_core/operations_v1/__init__.py +++ b/google/api_core/operations_v1/__init__.py @@ -14,11 +14,7 @@ """Package for interacting with the google.longrunning.operations meta-API.""" -import sys - +from google.api_core.operations_v1.operations_async_client import OperationsAsyncClient from google.api_core.operations_v1.operations_client import OperationsClient -__all__ = ["OperationsClient"] -if sys.version_info >= (3, 6, 0): - from google.api_core.operations_v1.operations_async_client import OperationsAsyncClient # noqa: F401 - __all__.append("OperationsAsyncClient") +__all__ = ["OperationsAsyncClient", "OperationsClient"] diff --git a/google/api_core/page_iterator.py b/google/api_core/page_iterator.py index 49879bc9..7ddc5cbc 100644 --- a/google/api_core/page_iterator.py +++ b/google/api_core/page_iterator.py @@ -81,8 +81,6 @@ import abc -import six - class Page(object): """Single page of results in an iterator. @@ -127,18 +125,15 @@ def __iter__(self): """The :class:`Page` is an iterator of items.""" return self - def next(self): + def __next__(self): """Get the next value in the page.""" - item = six.next(self._item_iter) + item = next(self._item_iter) result = self._item_to_value(self._parent, item) # Since we've successfully got the next value from the # iterator, we update the number of remaining. self._remaining -= 1 return result - # Alias needed for Python 2/3 support. - __next__ = next - def _item_to_value_identity(iterator, item): """An item to value transformer that returns the item un-changed.""" @@ -147,8 +142,7 @@ def _item_to_value_identity(iterator, item): return item -@six.add_metaclass(abc.ABCMeta) -class Iterator(object): +class Iterator(object, metaclass=abc.ABCMeta): """A generic class for iterating through API list responses. Args: @@ -235,9 +229,6 @@ def __next__(self): self.__active_iterator = iter(self) return next(self.__active_iterator) - # Preserve Python 2 compatibility. - next = __next__ - def _page_iter(self, increment): """Generator of pages of API responses. @@ -484,7 +475,7 @@ def _next_page(self): there are no pages left. """ try: - items = six.next(self._gax_page_iter) + items = next(self._gax_page_iter) page = Page(self, items, self.item_to_value) self.next_page_token = self._gax_page_iter.page_token or None return page diff --git a/google/api_core/path_template.py b/google/api_core/path_template.py index f202d40f..c5969c14 100644 --- a/google/api_core/path_template.py +++ b/google/api_core/path_template.py @@ -28,8 +28,6 @@ import functools import re -import six - # Regular expression for extracting variable parts from a path template. # The variables can be expressed as: # @@ -83,7 +81,7 @@ def _expand_variable_match(positional_vars, named_vars, match): name = match.group("name") if name is not None: try: - return six.text_type(named_vars[name]) + return str(named_vars[name]) except KeyError: raise ValueError( "Named variable '{}' not specified and needed by template " @@ -91,7 +89,7 @@ def _expand_variable_match(positional_vars, named_vars, match): ) elif positional is not None: try: - return six.text_type(positional_vars.pop(0)) + return str(positional_vars.pop(0)) except IndexError: raise ValueError( "Positional variable not specified and needed by template " diff --git a/google/api_core/protobuf_helpers.py b/google/api_core/protobuf_helpers.py index 8aff79aa..896e89c1 100644 --- a/google/api_core/protobuf_helpers.py +++ b/google/api_core/protobuf_helpers.py @@ -15,6 +15,7 @@ """Helpers for :mod:`protobuf`.""" import collections +import collections.abc import copy import inspect @@ -22,11 +23,6 @@ from google.protobuf import message from google.protobuf import wrappers_pb2 -try: - from collections import abc as collections_abc -except ImportError: # Python 2.7 - import collections as collections_abc - _SENTINEL = object() _WRAPPER_TYPES = ( @@ -179,7 +175,7 @@ def get(msg_or_dict, key, default=_SENTINEL): # If we get something else, complain. if isinstance(msg_or_dict, message.Message): answer = getattr(msg_or_dict, key, default) - elif isinstance(msg_or_dict, collections_abc.Mapping): + elif isinstance(msg_or_dict, collections.abc.Mapping): answer = msg_or_dict.get(key, default) else: raise TypeError( @@ -204,7 +200,7 @@ def _set_field_on_message(msg, key, value): """Set helper for protobuf Messages.""" # Attempt to set the value on the types of objects we know how to deal # with. - if isinstance(value, (collections_abc.MutableSequence, tuple)): + if isinstance(value, (collections.abc.MutableSequence, tuple)): # Clear the existing repeated protobuf message of any elements # currently inside it. while getattr(msg, key): @@ -212,13 +208,13 @@ def _set_field_on_message(msg, key, value): # Write our new elements to the repeated field. for item in value: - if isinstance(item, collections_abc.Mapping): + if isinstance(item, collections.abc.Mapping): getattr(msg, key).add(**item) else: # protobuf's RepeatedCompositeContainer doesn't support # append. getattr(msg, key).extend([item]) - elif isinstance(value, collections_abc.Mapping): + elif isinstance(value, collections.abc.Mapping): # Assign the dictionary values to the protobuf message. for item_key, item_value in value.items(): set(getattr(msg, key), item_key, item_value) @@ -241,7 +237,7 @@ def set(msg_or_dict, key, value): TypeError: If ``msg_or_dict`` is not a Message or dictionary. """ # Sanity check: Is our target object valid? - if not isinstance(msg_or_dict, (collections_abc.MutableMapping, message.Message)): + if not isinstance(msg_or_dict, (collections.abc.MutableMapping, message.Message)): raise TypeError( "set() expected a dict or protobuf message, got {!r}.".format( type(msg_or_dict) @@ -254,12 +250,12 @@ def set(msg_or_dict, key, value): # If a subkey exists, then get that object and call this method # recursively against it using the subkey. if subkey is not None: - if isinstance(msg_or_dict, collections_abc.MutableMapping): + if isinstance(msg_or_dict, collections.abc.MutableMapping): msg_or_dict.setdefault(basekey, {}) set(get(msg_or_dict, basekey), subkey, value) return - if isinstance(msg_or_dict, collections_abc.MutableMapping): + if isinstance(msg_or_dict, collections.abc.MutableMapping): msg_or_dict[key] = value else: _set_field_on_message(msg_or_dict, key, value) diff --git a/google/api_core/retry.py b/google/api_core/retry.py index 84967930..d39f97c1 100644 --- a/google/api_core/retry.py +++ b/google/api_core/retry.py @@ -63,11 +63,9 @@ def check_if_exists(): import time import requests.exceptions -import six from google.api_core import datetime_helpers from google.api_core import exceptions -from google.api_core import general_helpers from google.auth import exceptions as auth_exceptions _LOGGER = logging.getLogger(__name__) @@ -201,15 +199,12 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None): if deadline_datetime is not None: if deadline_datetime <= now: - six.raise_from( - exceptions.RetryError( - "Deadline of {:.1f}s exceeded while calling {}".format( - deadline, target - ), - last_exc, + raise exceptions.RetryError( + "Deadline of {:.1f}s exceeded while calling {}".format( + deadline, target ), last_exc, - ) + ) from last_exc else: time_to_deadline = (deadline_datetime - now).total_seconds() sleep = min(time_to_deadline, sleep) @@ -222,7 +217,6 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None): raise ValueError("Sleep generator stopped yielding sleep values.") -@six.python_2_unicode_compatible class Retry(object): """Exponential retry decorator. @@ -276,7 +270,7 @@ def __call__(self, func, on_error=None): if self._on_error is not None: on_error = self._on_error - @general_helpers.wraps(func) + @functools.wraps(func) def retry_wrapped_func(*args, **kwargs): """A wrapper that calls target function with retry.""" target = functools.partial(func, *args, **kwargs) diff --git a/google/api_core/timeout.py b/google/api_core/timeout.py index 17c1beab..73232180 100644 --- a/google/api_core/timeout.py +++ b/google/api_core/timeout.py @@ -54,11 +54,9 @@ def is_thing_ready(timeout=None): from __future__ import unicode_literals import datetime - -import six +import functools from google.api_core import datetime_helpers -from google.api_core import general_helpers _DEFAULT_INITIAL_TIMEOUT = 5.0 # seconds _DEFAULT_MAXIMUM_TIMEOUT = 30.0 # seconds @@ -68,7 +66,6 @@ def is_thing_ready(timeout=None): _DEFAULT_DEADLINE = None -@six.python_2_unicode_compatible class ConstantTimeout(object): """A decorator that adds a constant timeout argument. @@ -95,7 +92,7 @@ def __call__(self, func): Callable: The wrapped function. """ - @general_helpers.wraps(func) + @functools.wraps(func) def func_with_timeout(*args, **kwargs): """Wrapped function that adds timeout.""" kwargs["timeout"] = self._timeout @@ -140,7 +137,6 @@ def _exponential_timeout_generator(initial, maximum, multiplier, deadline): timeout = timeout * multiplier -@six.python_2_unicode_compatible class ExponentialTimeout(object): """A decorator that adds an exponentially increasing timeout argument. @@ -207,7 +203,7 @@ def __call__(self, func): self._initial, self._maximum, self._multiplier, self._deadline ) - @general_helpers.wraps(func) + @functools.wraps(func) def func_with_timeout(*args, **kwargs): """Wrapped function that adds timeout.""" kwargs["timeout"] = next(timeouts) diff --git a/google/api_core/version.py b/google/api_core/version.py index 5fd0b133..c9cdad6f 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.31.1" +__version__ = "2.0.0-b1" diff --git a/noxfile.py b/noxfile.py index 2f11137d..a8f464e0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -111,13 +111,13 @@ def default(session): session.run(*pytest_args) -@nox.session(python=["2.7", "3.6", "3.7", "3.8", "3.9"]) +@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) def unit(session): """Run the unit test suite.""" default(session) -@nox.session(python=["2.7", "3.6", "3.7", "3.8", "3.9"]) +@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) def unit_grpc_gcp(session): """Run the unit test suite with grpcio-gcp installed.""" constraints_path = str( @@ -137,7 +137,6 @@ def lint_setup_py(session): session.run("python", "setup.py", "check", "--restructuredtext", "--strict") -# No 2.7 due to https://github.com/google/importlab/issues/26. # No 3.7 because pytype supports up to 3.6 only. @nox.session(python="3.6") def pytype(session): diff --git a/owlbot.py b/owlbot.py index 5c590e4c..451f7c48 100644 --- a/owlbot.py +++ b/owlbot.py @@ -22,8 +22,15 @@ # ---------------------------------------------------------------------------- # Add templated files # ---------------------------------------------------------------------------- -templated_files = common.py_library(cov_level=100) -s.move(templated_files, excludes=["noxfile.py", ".flake8", ".coveragerc", "setup.cfg"]) +excludes = [ + "noxfile.py", # pytype + "setup.cfg", # pytype + ".flake8", # flake8-import-order, layout + ".coveragerc", # layout + "CONTRIBUTING.rst", # no systests +] +templated_files = common.py_library(microgenerator=True, cov_level=100) +s.move(templated_files, excludes=excludes) # Add pytype support s.replace( diff --git a/setup.py b/setup.py index 26fab7e6..d98c69c5 100644 --- a/setup.py +++ b/setup.py @@ -31,12 +31,9 @@ dependencies = [ "googleapis-common-protos >= 1.6.0, < 2.0dev", "protobuf >= 3.12.0", - "google-auth >= 1.25.0, < 2.0dev", + "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", "setuptools >= 40.3.0", - "packaging >= 14.3", - "six >= 1.13.0", - "pytz", 'futures >= 3.2.0; python_version < "3.2"', ] extras = { @@ -86,10 +83,7 @@ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", @@ -102,7 +96,7 @@ namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, - python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*", + python_requires=">=3.6", include_package_data=True, zip_safe=False, ) diff --git a/testing/constraints-2.7.txt b/testing/constraints-2.7.txt deleted file mode 100644 index 246c89d5..00000000 --- a/testing/constraints-2.7.txt +++ /dev/null @@ -1 +0,0 @@ -googleapis-common-protos >= 1.6.0, < 1.53dev diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index ff5b4a7f..2544e8e5 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -11,7 +11,6 @@ google-auth==1.25.0 requests==2.18.0 setuptools==40.3.0 packaging==14.3 -six==1.13.0 grpcio==1.29.0 grpcio-gcp==0.2.2 grpcio-gcp==0.2.2 diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index 602d640f..b876d9ad 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -14,12 +14,12 @@ import datetime import logging +import queue import threading import grpc import mock import pytest -from six.moves import queue from google.api_core import bidi from google.api_core import exceptions @@ -221,18 +221,12 @@ def cancel_side_effect(): class ClosedCall(object): - # NOTE: This is needed because defining `.next` on an **instance** - # rather than the **class** will not be iterable in Python 2. - # This is problematic since a `Mock` just sets members. - def __init__(self, exception): self.exception = exception def __next__(self): raise self.exception - next = __next__ # Python 2 - def is_active(self): return False @@ -354,8 +348,6 @@ def __next__(self): raise item return item - next = __next__ # Python 2 - def is_active(self): return self._is_active diff --git a/tests/unit/test_datetime_helpers.py b/tests/unit/test_datetime_helpers.py index 4ddcf361..5f5470a6 100644 --- a/tests/unit/test_datetime_helpers.py +++ b/tests/unit/test_datetime_helpers.py @@ -16,7 +16,6 @@ import datetime import pytest -import pytz from google.api_core import datetime_helpers from google.protobuf import timestamp_pb2 @@ -31,7 +30,7 @@ def test_utcnow(): def test_to_milliseconds(): - dt = datetime.datetime(1970, 1, 1, 0, 0, 1, tzinfo=pytz.utc) + dt = datetime.datetime(1970, 1, 1, 0, 0, 1, tzinfo=datetime.timezone.utc) assert datetime_helpers.to_milliseconds(dt) == 1000 @@ -42,7 +41,7 @@ def test_to_microseconds(): def test_to_microseconds_non_utc(): - zone = pytz.FixedOffset(-1) + zone = datetime.timezone(datetime.timedelta(minutes=-1)) dt = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=zone) assert datetime_helpers.to_microseconds(dt) == ONE_MINUTE_IN_MICROSECONDS @@ -56,7 +55,7 @@ def test_to_microseconds_naive(): def test_from_microseconds(): five_mins_from_epoch_in_microseconds = 5 * ONE_MINUTE_IN_MICROSECONDS five_mins_from_epoch_datetime = datetime.datetime( - 1970, 1, 1, 0, 5, 0, tzinfo=pytz.utc + 1970, 1, 1, 0, 5, 0, tzinfo=datetime.timezone.utc ) result = datetime_helpers.from_microseconds(five_mins_from_epoch_in_microseconds) @@ -78,28 +77,28 @@ def test_from_iso8601_time(): def test_from_rfc3339(): value = "2009-12-17T12:44:32.123456Z" assert datetime_helpers.from_rfc3339(value) == datetime.datetime( - 2009, 12, 17, 12, 44, 32, 123456, pytz.utc + 2009, 12, 17, 12, 44, 32, 123456, datetime.timezone.utc ) def test_from_rfc3339_nanos(): value = "2009-12-17T12:44:32.123456Z" assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( - 2009, 12, 17, 12, 44, 32, 123456, pytz.utc + 2009, 12, 17, 12, 44, 32, 123456, datetime.timezone.utc ) def test_from_rfc3339_without_nanos(): value = "2009-12-17T12:44:32Z" assert datetime_helpers.from_rfc3339(value) == datetime.datetime( - 2009, 12, 17, 12, 44, 32, 0, pytz.utc + 2009, 12, 17, 12, 44, 32, 0, datetime.timezone.utc ) def test_from_rfc3339_nanos_without_nanos(): value = "2009-12-17T12:44:32Z" assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( - 2009, 12, 17, 12, 44, 32, 0, pytz.utc + 2009, 12, 17, 12, 44, 32, 0, datetime.timezone.utc ) @@ -119,7 +118,7 @@ def test_from_rfc3339_nanos_without_nanos(): def test_from_rfc3339_with_truncated_nanos(truncated, micros): value = "2009-12-17T12:44:32.{}Z".format(truncated) assert datetime_helpers.from_rfc3339(value) == datetime.datetime( - 2009, 12, 17, 12, 44, 32, micros, pytz.utc + 2009, 12, 17, 12, 44, 32, micros, datetime.timezone.utc ) @@ -148,7 +147,7 @@ def test_from_rfc3339_nanos_is_deprecated(): def test_from_rfc3339_nanos_with_truncated_nanos(truncated, micros): value = "2009-12-17T12:44:32.{}Z".format(truncated) assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( - 2009, 12, 17, 12, 44, 32, micros, pytz.utc + 2009, 12, 17, 12, 44, 32, micros, datetime.timezone.utc ) @@ -171,20 +170,20 @@ def test_to_rfc3339(): def test_to_rfc3339_with_utc(): - value = datetime.datetime(2016, 4, 5, 13, 30, 0, tzinfo=pytz.utc) + value = datetime.datetime(2016, 4, 5, 13, 30, 0, tzinfo=datetime.timezone.utc) expected = "2016-04-05T13:30:00.000000Z" assert datetime_helpers.to_rfc3339(value, ignore_zone=False) == expected def test_to_rfc3339_with_non_utc(): - zone = pytz.FixedOffset(-60) + zone = datetime.timezone(datetime.timedelta(minutes=-60)) value = datetime.datetime(2016, 4, 5, 13, 30, 0, tzinfo=zone) expected = "2016-04-05T14:30:00.000000Z" assert datetime_helpers.to_rfc3339(value, ignore_zone=False) == expected def test_to_rfc3339_with_non_utc_ignore_zone(): - zone = pytz.FixedOffset(-60) + zone = datetime.timezone(datetime.timedelta(minutes=-60)) value = datetime.datetime(2016, 4, 5, 13, 30, 0, tzinfo=zone) expected = "2016-04-05T13:30:00.000000Z" assert datetime_helpers.to_rfc3339(value, ignore_zone=True) == expected @@ -283,7 +282,7 @@ def test_from_rfc3339_w_invalid(): def test_from_rfc3339_wo_fraction(): timestamp = "2016-12-20T21:13:47Z" expected = datetime_helpers.DatetimeWithNanoseconds( - 2016, 12, 20, 21, 13, 47, tzinfo=pytz.UTC + 2016, 12, 20, 21, 13, 47, tzinfo=datetime.timezone.utc ) stamp = datetime_helpers.DatetimeWithNanoseconds.from_rfc3339(timestamp) assert stamp == expected @@ -292,7 +291,7 @@ def test_from_rfc3339_wo_fraction(): def test_from_rfc3339_w_partial_precision(): timestamp = "2016-12-20T21:13:47.1Z" expected = datetime_helpers.DatetimeWithNanoseconds( - 2016, 12, 20, 21, 13, 47, microsecond=100000, tzinfo=pytz.UTC + 2016, 12, 20, 21, 13, 47, microsecond=100000, tzinfo=datetime.timezone.utc ) stamp = datetime_helpers.DatetimeWithNanoseconds.from_rfc3339(timestamp) assert stamp == expected @@ -301,7 +300,7 @@ def test_from_rfc3339_w_partial_precision(): def test_from_rfc3339_w_full_precision(): timestamp = "2016-12-20T21:13:47.123456789Z" expected = datetime_helpers.DatetimeWithNanoseconds( - 2016, 12, 20, 21, 13, 47, nanosecond=123456789, tzinfo=pytz.UTC + 2016, 12, 20, 21, 13, 47, nanosecond=123456789, tzinfo=datetime.timezone.utc ) stamp = datetime_helpers.DatetimeWithNanoseconds.from_rfc3339(timestamp) assert stamp == expected @@ -332,7 +331,9 @@ def test_timestamp_pb_wo_nanos_naive(): stamp = datetime_helpers.DatetimeWithNanoseconds( 2016, 12, 20, 21, 13, 47, 123456 ) - delta = stamp.replace(tzinfo=pytz.UTC) - datetime_helpers._UTC_EPOCH + delta = ( + stamp.replace(tzinfo=datetime.timezone.utc) - datetime_helpers._UTC_EPOCH + ) seconds = int(delta.total_seconds()) nanos = 123456000 timestamp = timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos) @@ -341,7 +342,7 @@ def test_timestamp_pb_wo_nanos_naive(): @staticmethod def test_timestamp_pb_w_nanos(): stamp = datetime_helpers.DatetimeWithNanoseconds( - 2016, 12, 20, 21, 13, 47, nanosecond=123456789, tzinfo=pytz.UTC + 2016, 12, 20, 21, 13, 47, nanosecond=123456789, tzinfo=datetime.timezone.utc ) delta = stamp - datetime_helpers._UTC_EPOCH timestamp = timestamp_pb2.Timestamp( @@ -351,7 +352,9 @@ def test_timestamp_pb_w_nanos(): @staticmethod def test_from_timestamp_pb_wo_nanos(): - when = datetime.datetime(2016, 12, 20, 21, 13, 47, 123456, tzinfo=pytz.UTC) + when = datetime.datetime( + 2016, 12, 20, 21, 13, 47, 123456, tzinfo=datetime.timezone.utc + ) delta = when - datetime_helpers._UTC_EPOCH seconds = int(delta.total_seconds()) timestamp = timestamp_pb2.Timestamp(seconds=seconds) @@ -361,11 +364,13 @@ def test_from_timestamp_pb_wo_nanos(): assert _to_seconds(when) == _to_seconds(stamp) assert stamp.microsecond == 0 assert stamp.nanosecond == 0 - assert stamp.tzinfo == pytz.UTC + assert stamp.tzinfo == datetime.timezone.utc @staticmethod def test_from_timestamp_pb_w_nanos(): - when = datetime.datetime(2016, 12, 20, 21, 13, 47, 123456, tzinfo=pytz.UTC) + when = datetime.datetime( + 2016, 12, 20, 21, 13, 47, 123456, tzinfo=datetime.timezone.utc + ) delta = when - datetime_helpers._UTC_EPOCH seconds = int(delta.total_seconds()) timestamp = timestamp_pb2.Timestamp(seconds=seconds, nanos=123456789) @@ -375,7 +380,7 @@ def test_from_timestamp_pb_w_nanos(): assert _to_seconds(when) == _to_seconds(stamp) assert stamp.microsecond == 123456 assert stamp.nanosecond == 123456789 - assert stamp.tzinfo == pytz.UTC + assert stamp.tzinfo == datetime.timezone.utc def _to_seconds(value): @@ -387,5 +392,5 @@ def _to_seconds(value): Returns: int: Microseconds since the unix epoch. """ - assert value.tzinfo is pytz.UTC + assert value.tzinfo is datetime.timezone.utc return calendar.timegm(value.timetuple()) diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index fb29015f..10599457 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import http.client import json import grpc import mock import requests -from six.moves import http_client from google.api_core import exceptions @@ -50,8 +50,8 @@ def test_create_google_cloud_error_with_args(): def test_from_http_status(): message = "message" - exception = exceptions.from_http_status(http_client.NOT_FOUND, message) - assert exception.code == http_client.NOT_FOUND + exception = exceptions.from_http_status(http.client.NOT_FOUND, message) + assert exception.code == http.client.NOT_FOUND assert exception.message == message assert exception.errors == [] @@ -61,11 +61,11 @@ def test_from_http_status_with_errors_and_response(): errors = ["1", "2"] response = mock.sentinel.response exception = exceptions.from_http_status( - http_client.NOT_FOUND, message, errors=errors, response=response + http.client.NOT_FOUND, message, errors=errors, response=response ) assert isinstance(exception, exceptions.NotFound) - assert exception.code == http_client.NOT_FOUND + assert exception.code == http.client.NOT_FOUND assert exception.message == message assert exception.errors == errors assert exception.response == response @@ -82,7 +82,7 @@ def test_from_http_status_unknown_code(): def make_response(content): response = requests.Response() response._content = content - response.status_code = http_client.NOT_FOUND + response.status_code = http.client.NOT_FOUND response.request = requests.Request( method="POST", url="https://example.com" ).prepare() @@ -95,7 +95,7 @@ def test_from_http_response_no_content(): exception = exceptions.from_http_response(response) assert isinstance(exception, exceptions.NotFound) - assert exception.code == http_client.NOT_FOUND + assert exception.code == http.client.NOT_FOUND assert exception.message == "POST https://example.com/: unknown error" assert exception.response == response @@ -106,7 +106,7 @@ def test_from_http_response_text_content(): exception = exceptions.from_http_response(response) assert isinstance(exception, exceptions.NotFound) - assert exception.code == http_client.NOT_FOUND + assert exception.code == http.client.NOT_FOUND assert exception.message == "POST https://example.com/: message" @@ -120,7 +120,7 @@ def test_from_http_response_json_content(): exception = exceptions.from_http_response(response) assert isinstance(exception, exceptions.NotFound) - assert exception.code == http_client.NOT_FOUND + assert exception.code == http.client.NOT_FOUND assert exception.message == "POST https://example.com/: json message" assert exception.errors == ["1", "2"] @@ -131,22 +131,22 @@ def test_from_http_response_bad_json_content(): exception = exceptions.from_http_response(response) assert isinstance(exception, exceptions.NotFound) - assert exception.code == http_client.NOT_FOUND + assert exception.code == http.client.NOT_FOUND assert exception.message == "POST https://example.com/: unknown error" def test_from_http_response_json_unicode_content(): response = make_response( json.dumps( - {"error": {"message": u"\u2019 message", "errors": ["1", "2"]}} + {"error": {"message": "\u2019 message", "errors": ["1", "2"]}} ).encode("utf-8") ) exception = exceptions.from_http_response(response) assert isinstance(exception, exceptions.NotFound) - assert exception.code == http_client.NOT_FOUND - assert exception.message == u"POST https://example.com/: \u2019 message" + assert exception.code == http.client.NOT_FOUND + assert exception.message == "POST https://example.com/: \u2019 message" assert exception.errors == ["1", "2"] @@ -155,7 +155,7 @@ def test_from_grpc_status(): exception = exceptions.from_grpc_status(grpc.StatusCode.OUT_OF_RANGE, message) assert isinstance(exception, exceptions.BadRequest) assert isinstance(exception, exceptions.OutOfRange) - assert exception.code == http_client.BAD_REQUEST + assert exception.code == http.client.BAD_REQUEST assert exception.grpc_status_code == grpc.StatusCode.OUT_OF_RANGE assert exception.message == message assert exception.errors == [] @@ -166,7 +166,7 @@ def test_from_grpc_status_as_int(): exception = exceptions.from_grpc_status(11, message) assert isinstance(exception, exceptions.BadRequest) assert isinstance(exception, exceptions.OutOfRange) - assert exception.code == http_client.BAD_REQUEST + assert exception.code == http.client.BAD_REQUEST assert exception.grpc_status_code == grpc.StatusCode.OUT_OF_RANGE assert exception.message == message assert exception.errors == [] @@ -203,7 +203,7 @@ def test_from_grpc_error(): assert isinstance(exception, exceptions.BadRequest) assert isinstance(exception, exceptions.InvalidArgument) - assert exception.code == http_client.BAD_REQUEST + assert exception.code == http.client.BAD_REQUEST assert exception.grpc_status_code == grpc.StatusCode.INVALID_ARGUMENT assert exception.message == message assert exception.errors == [error] diff --git a/tests/unit/test_general_helpers.py b/tests/unit/test_general_helpers.py deleted file mode 100644 index 027d4892..00000000 --- a/tests/unit/test_general_helpers.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2017, Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import functools - -from google.api_core import general_helpers - - -def test_wraps_normal_func(): - def func(): - return 42 - - @general_helpers.wraps(func) - def replacement(): - return func() - - assert replacement() == 42 - - -def test_wraps_partial(): - def func(): - return 42 - - partial = functools.partial(func) - - @general_helpers.wraps(partial) - def replacement(): - return func() - - assert replacement() == 42 diff --git a/tests/unit/test_page_iterator.py b/tests/unit/test_page_iterator.py index 668cf392..a44e998b 100644 --- a/tests/unit/test_page_iterator.py +++ b/tests/unit/test_page_iterator.py @@ -17,7 +17,6 @@ import mock import pytest -import six from google.api_core import page_iterator @@ -56,17 +55,17 @@ def test_iterator_calls_parent_item_to_value(self): assert item_to_value.call_count == 0 assert page.remaining == 100 - assert six.next(page) == 10 + assert next(page) == 10 assert item_to_value.call_count == 1 item_to_value.assert_called_with(parent, 10) assert page.remaining == 99 - assert six.next(page) == 11 + assert next(page) == 11 assert item_to_value.call_count == 2 item_to_value.assert_called_with(parent, 11) assert page.remaining == 98 - assert six.next(page) == 12 + assert next(page) == 12 assert item_to_value.call_count == 3 item_to_value.assert_called_with(parent, 12) assert page.remaining == 97 @@ -197,17 +196,17 @@ def test__items_iter(self): # Consume items and check the state of the iterator. assert iterator.num_results == 0 - assert six.next(items_iter) == item1 + assert next(items_iter) == item1 assert iterator.num_results == 1 - assert six.next(items_iter) == item2 + assert next(items_iter) == item2 assert iterator.num_results == 2 - assert six.next(items_iter) == item3 + assert next(items_iter) == item3 assert iterator.num_results == 3 with pytest.raises(StopIteration): - six.next(items_iter) + next(items_iter) def test___iter__(self): iterator = PageIteratorImpl(None, None) @@ -289,16 +288,16 @@ def test_iterate(self): items_iter = iter(iterator) - val1 = six.next(items_iter) + val1 = next(items_iter) assert val1 == item1 assert iterator.num_results == 1 - val2 = six.next(items_iter) + val2 = next(items_iter) assert val2 == item2 assert iterator.num_results == 2 with pytest.raises(StopIteration): - six.next(items_iter) + next(items_iter) api_request.assert_called_once_with(method="GET", path=path, query_params={}) @@ -503,7 +502,7 @@ def api_request(*args, **kw): items_iter = iter(iterator.pages) npages = int(math.ceil(float(n_results) / page_size)) for ipage in range(npages): - assert list(six.next(items_iter)) == [ + assert list(next(items_iter)) == [ dict(name=str(i)) for i in range( ipage * page_size, min((ipage + 1) * page_size, n_results), @@ -512,11 +511,11 @@ def api_request(*args, **kw): else: items_iter = iter(iterator) for i in range(n_results): - assert six.next(items_iter) == dict(name=str(i)) + assert next(items_iter) == dict(name=str(i)) assert iterator.num_results == i + 1 with pytest.raises(StopIteration): - six.next(items_iter) + next(items_iter) class TestGRPCIterator(object): @@ -621,7 +620,7 @@ def __init__(self, pages, page_token=None): self.page_token = page_token def next(self): - return six.next(self._pages) + return next(self._pages) __next__ = next From d566f2893eb708c109ebc3b4fa16f57aa4c65f26 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 3 Aug 2021 14:02:27 -0400 Subject: [PATCH 003/112] chore: release 2.0.0b1 (#243) --- CHANGELOG.md | 11 +++++++++++ google/api_core/version.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e00e424c..e8e5e7a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.0.0b1](https://www.github.com/googleapis/python-api-core/compare/v1.31.1...v2.0.0b1) (2021-08-03) + + +### ⚠ BREAKING CHANGES + +* drop support for Python 2.7 / 3.5 ([#212](https://www.github.com/googleapis/python-api-core/issues/212)) ([a30f004](https://www.github.com/googleapis/python-api-core/commit/a30f004e74f709d46e905dd819c71f43354e9ac9)) + +### Bug Fixes + +* strip trailing _ from field mask paths ([#228](https://www.github.com/googleapis/python-api-core/issues/228)) ([ff6ef1b](https://www.github.com/googleapis/python-api-core/commit/ff6ef1bd07fa68307b7c82c910416d770e7b3416)) + ### [1.31.1](https://www.github.com/googleapis/python-api-core/compare/v1.31.0...v1.31.1) (2021-07-26) diff --git a/google/api_core/version.py b/google/api_core/version.py index c9cdad6f..5fa77444 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.0.0-b1" +__version__ = "2.0.0b1" From 6789e675814983cbd413dc045c11a7c7a176042c Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Fri, 6 Aug 2021 10:52:12 -0600 Subject: [PATCH 004/112] chore: remove non-custom rules from sync-repo-settings (#241) See https://github.com/googleapis/repo-automation-bots/blob/63c858e539e1f4d9bb8ea66e12f9c0a0de5fef55/packages/sync-repo-settings/src/required-checks.json#L40-L50 for defaults. --- .github/sync-repo-settings.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 4a559eaf..e621885d 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -1,12 +1,3 @@ -rebaseMergeAllowed: true -squashMergeAllowed: true -mergeCommitAllowed: false -branchProtectionRules: -- pattern: master - isAdminEnforced: true - requiredApprovingReviewCount: 1 - requiresCodeOwnerReviews: true - requiresStrictStatusChecks: true permissionRules: - team: actools-python permission: admin From 258ba4a0119dbd1426afcd79d2f549b4c0fb6ec6 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 12 Aug 2021 15:55:59 +0000 Subject: [PATCH 005/112] chore(python): avoid `.nox` directories when building docs (#249) Source-Link: https://github.com/googleapis/synthtool/commit/7e1f6da50524b5d98eb67adbf6dd0805df54233d Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:a1a891041baa4ffbe1a809ac1b8b9b4a71887293c9101c88e8e255943c5aec2d --- .github/.OwlBot.lock.yaml | 2 +- docs/conf.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 9ee60f7e..b771c37c 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:aea14a583128771ae8aefa364e1652f3c56070168ef31beb203534222d842b8b + digest: sha256:a1a891041baa4ffbe1a809ac1b8b9b4a71887293c9101c88e8e255943c5aec2d diff --git a/docs/conf.py b/docs/conf.py index 93516048..aec958f3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -110,6 +110,7 @@ # directories to ignore when looking for source files. exclude_patterns = [ "_build", + "**/.nox/**/*", "samples/AUTHORING_GUIDE.md", "samples/CONTRIBUTING.md", "samples/snippets/README.rst", From ffa528e088dc5e426ed5652be022c8fb43834e25 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 12 Aug 2021 17:42:46 -0400 Subject: [PATCH 006/112] chore: avoid duplicating pins of grpcio in noxfile (#246) Rely on the pins in 'setup.py' as the Source of Truth. See https://github.com/googleapis/python-api-core/pull/234#pullrequestreview-724669326 --- noxfile.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/noxfile.py b/noxfile.py index a8f464e0..84470f5a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -80,8 +80,8 @@ def default(session): ) # Install all test dependencies, then install this package in-place. - session.install("mock", "pytest", "pytest-cov", "grpcio >= 1.0.2") - session.install("-e", ".", "-c", constraints_path) + session.install("mock", "pytest", "pytest-cov") + session.install("-e", ".[grpc]", "-c", constraints_path) pytest_args = [ "python", @@ -124,7 +124,7 @@ def unit_grpc_gcp(session): CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) # Install grpcio-gcp - session.install("grpcio-gcp", "-c", constraints_path) + session.install("-e", ".[grpcgcp]", "-c", constraints_path) default(session) @@ -141,9 +141,7 @@ def lint_setup_py(session): @nox.session(python="3.6") def pytype(session): """Run type-checking.""" - session.install( - ".", "grpcio >= 1.8.2", "grpcio-gcp >= 0.2.2", "pytype >= 2019.3.21" - ) + session.install(".[grpc, grpcgcp]", "pytype >= 2019.3.21") session.run("pytype") @@ -163,8 +161,7 @@ def cover(session): def docs(session): """Build the docs for this library.""" - session.install(".", "grpcio >= 1.8.2", "grpcio-gcp >= 0.2.2") - session.install("-e", ".") + session.install("-e", ".[grpc, grpcgcp]") session.install("sphinx==4.0.1", "alabaster", "recommonmark") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) From a54cc10bdcf8dd8df59d24c58a9b9053d456c846 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 13 Aug 2021 11:08:59 -0400 Subject: [PATCH 007/112] chore: drop mention of Python 2.7 from templates (#252) Source-Link: https://github.com/googleapis/synthtool/commit/facee4cc1ea096cd8bcc008bb85929daa7c414c0 Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:9743664022bd63a8084be67f144898314c7ca12f0a03e422ac17c733c129d803 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- scripts/readme-gen/templates/install_deps.tmpl.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index b771c37c..a9fcd07c 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:a1a891041baa4ffbe1a809ac1b8b9b4a71887293c9101c88e8e255943c5aec2d + digest: sha256:9743664022bd63a8084be67f144898314c7ca12f0a03e422ac17c733c129d803 diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst index a0406dba..275d6498 100644 --- a/scripts/readme-gen/templates/install_deps.tmpl.rst +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -12,7 +12,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. +#. Create a virtualenv. Samples are compatible with Python 3.6+. .. code-block:: bash From 362ca6ce33075f7e5f5696b2116401162babf6da Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 13 Aug 2021 12:04:15 -0400 Subject: [PATCH 008/112] chore: update if_transient_error docs to match behaviour (#253) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #185 🦕 --- google/api_core/retry.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/google/api_core/retry.py b/google/api_core/retry.py index d39f97c1..44cd7469 100644 --- a/google/api_core/retry.py +++ b/google/api_core/retry.py @@ -113,8 +113,11 @@ def if_exception_type_predicate(exception): ``INTERNAL(13)`` and its subclasses. - :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429 - :class:`google.api_core.exceptions.ServiceUnavailable` - HTTP 503 -- :class:`google.api_core.exceptions.ResourceExhausted` - gRPC - ``RESOURCE_EXHAUSTED(8)`` +- :class:`requests.exceptions.ConnectionError` +- :class:`requests.exceptions.ChunkedEncodingError` - The server declared + chunked encoding but sent an invalid chunk. +- :class:`google.auth.exceptions.TransportError` - Used to indicate an + error occurred during an HTTP request. """ # pylint: enable=invalid-name From bdbf889210b709d7c1945f2160bcba9161b4dd2e Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 13 Aug 2021 09:05:01 -0700 Subject: [PATCH 009/112] fix: bump grpcio version to use stable aio API (#234) --- google/api_core/grpc_helpers_async.py | 2 +- setup.py | 2 +- testing/constraints-3.6.txt | 2 +- tests/asyncio/gapic/test_method_async.py | 2 +- .../test_operations_async_client.py | 2 +- tests/asyncio/test_grpc_helpers_async.py | 28 +++++++++---------- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/google/api_core/grpc_helpers_async.py b/google/api_core/grpc_helpers_async.py index 92df645b..452e7871 100644 --- a/google/api_core/grpc_helpers_async.py +++ b/google/api_core/grpc_helpers_async.py @@ -22,7 +22,7 @@ import functools import grpc -from grpc.experimental import aio +from grpc import aio from google.api_core import exceptions, grpc_helpers diff --git a/setup.py b/setup.py index d98c69c5..cade3d82 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ 'futures >= 3.2.0; python_version < "3.2"', ] extras = { - "grpc": "grpcio >= 1.29.0, < 2.0dev", + "grpc": "grpcio >= 1.33.2, < 2.0dev", "grpcgcp": "grpcio-gcp >= 0.2.2", "grpcio-gcp": "grpcio-gcp >= 0.2.2", } diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 2544e8e5..744e991b 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -11,6 +11,6 @@ google-auth==1.25.0 requests==2.18.0 setuptools==40.3.0 packaging==14.3 -grpcio==1.29.0 +grpcio==1.33.2 grpcio-gcp==0.2.2 grpcio-gcp==0.2.2 diff --git a/tests/asyncio/gapic/test_method_async.py b/tests/asyncio/gapic/test_method_async.py index 2c6bbab9..e24fe444 100644 --- a/tests/asyncio/gapic/test_method_async.py +++ b/tests/asyncio/gapic/test_method_async.py @@ -14,7 +14,7 @@ import datetime -from grpc.experimental import aio +from grpc import aio import mock import pytest diff --git a/tests/asyncio/operations_v1/test_operations_async_client.py b/tests/asyncio/operations_v1/test_operations_async_client.py index 5473e8ae..3fb8427b 100644 --- a/tests/asyncio/operations_v1/test_operations_async_client.py +++ b/tests/asyncio/operations_v1/test_operations_async_client.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from grpc.experimental import aio +from grpc import aio import mock import pytest diff --git a/tests/asyncio/test_grpc_helpers_async.py b/tests/asyncio/test_grpc_helpers_async.py index a511ed46..f0b6c5f1 100644 --- a/tests/asyncio/test_grpc_helpers_async.py +++ b/tests/asyncio/test_grpc_helpers_async.py @@ -13,7 +13,7 @@ # limitations under the License. import grpc -from grpc.experimental import aio +from grpc import aio import mock import pytest @@ -270,7 +270,7 @@ def test_wrap_errors_streaming(wrap_stream_errors): autospec=True, return_value=(mock.sentinel.credentials, mock.sentinel.projet), ) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_call): target = "example.com:443" composite_creds = composite_creds_call.return_value @@ -295,7 +295,7 @@ def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_c autospec=True, return_value=(mock.sentinel.credentials, mock.sentinel.projet), ) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_implicit_with_default_host( grpc_secure_channel, default, composite_creds_call, request, auth_metadata_plugin ): @@ -319,7 +319,7 @@ def test_create_channel_implicit_with_default_host( "google.auth.default", return_value=(mock.sentinel.credentials, mock.sentinel.projet), ) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_implicit_with_ssl_creds( grpc_secure_channel, default, composite_creds_call ): @@ -341,7 +341,7 @@ def test_create_channel_implicit_with_ssl_creds( autospec=True, return_value=(mock.sentinel.credentials, mock.sentinel.projet), ) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_implicit_with_scopes( grpc_secure_channel, default, composite_creds_call ): @@ -362,7 +362,7 @@ def test_create_channel_implicit_with_scopes( autospec=True, return_value=(mock.sentinel.credentials, mock.sentinel.projet), ) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_implicit_with_default_scopes( grpc_secure_channel, default, composite_creds_call ): @@ -394,7 +394,7 @@ def test_create_channel_explicit_with_duplicate_credentials(): @mock.patch("grpc.composite_channel_credentials") @mock.patch("google.auth.credentials.with_scopes_if_required", autospec=True) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_creds_call): target = "example.com:443" composite_creds = composite_creds_call.return_value @@ -411,7 +411,7 @@ def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_cred @mock.patch("grpc.composite_channel_credentials") -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_call): target = "example.com:443" scopes = ["1", "2"] @@ -430,7 +430,7 @@ def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_cal @mock.patch("grpc.composite_channel_credentials") -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_explicit_default_scopes( grpc_secure_channel, composite_creds_call ): @@ -453,7 +453,7 @@ def test_create_channel_explicit_default_scopes( @mock.patch("grpc.composite_channel_credentials") -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_explicit_with_quota_project( grpc_secure_channel, composite_creds_call ): @@ -474,7 +474,7 @@ def test_create_channel_explicit_with_quota_project( @mock.patch("grpc.composite_channel_credentials") -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") @mock.patch( "google.auth.load_credentials_from_file", autospec=True, @@ -500,7 +500,7 @@ def test_create_channnel_with_credentials_file( @mock.patch("grpc.composite_channel_credentials") -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") @mock.patch( "google.auth.load_credentials_from_file", autospec=True, @@ -527,7 +527,7 @@ def test_create_channel_with_credentials_file_and_scopes( @mock.patch("grpc.composite_channel_credentials") -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") @mock.patch( "google.auth.load_credentials_from_file", autospec=True, @@ -556,7 +556,7 @@ def test_create_channel_with_credentials_file_and_default_scopes( @pytest.mark.skipif( grpc_helpers_async.HAS_GRPC_GCP, reason="grpc_gcp module not available" ) -@mock.patch("grpc.experimental.aio.secure_channel") +@mock.patch("grpc.aio.secure_channel") def test_create_channel_without_grpc_gcp(grpc_secure_channel): target = "example.com:443" scopes = ["test_scope"] From bb3b26d54dbe6acca1e105039b2fb405253980c0 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 18 Aug 2021 12:08:52 -0400 Subject: [PATCH 010/112] tests: silence a warning from 'charset_normalizer' (#247) --- google/api_core/retry.py | 4 ++-- tests/unit/test_exceptions.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/google/api_core/retry.py b/google/api_core/retry.py index 44cd7469..bd3a4a65 100644 --- a/google/api_core/retry.py +++ b/google/api_core/retry.py @@ -114,9 +114,9 @@ def if_exception_type_predicate(exception): - :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429 - :class:`google.api_core.exceptions.ServiceUnavailable` - HTTP 503 - :class:`requests.exceptions.ConnectionError` -- :class:`requests.exceptions.ChunkedEncodingError` - The server declared +- :class:`requests.exceptions.ChunkedEncodingError` - The server declared chunked encoding but sent an invalid chunk. -- :class:`google.auth.exceptions.TransportError` - Used to indicate an +- :class:`google.auth.exceptions.TransportError` - Used to indicate an error occurred during an HTTP request. """ # pylint: enable=invalid-name diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 10599457..e9709f26 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -102,6 +102,7 @@ def test_from_http_response_no_content(): def test_from_http_response_text_content(): response = make_response(b"message") + response.encoding = "UTF8" # suppress charset_normalizer warning exception = exceptions.from_http_response(response) From 40f52bf1100cf56b7f9af267d210b8a72fc34f08 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 18 Aug 2021 16:27:49 -0600 Subject: [PATCH 011/112] chore: release 2.0.0 (#254) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit :robot: I have created a release \*beep\* \*boop\* --- ## [2.0.0](https://www.github.com/googleapis/python-api-core/compare/v2.0.0-b1...v2.0.0) (2021-08-18) ### ⚠ BREAKING CHANGES * drop support for Python 2.7 / 3.5 ([#212](https://www.github.com/googleapis/python-api-core/issues/212)) ([a30f004](https://www.github.com/googleapis/python-api-core/commit/a30f004e74f709d46e905dd819c71f43354e9ac9)) ### Bug Fixes * bump grpcio version to use stable aio API ([#234](https://www.github.com/googleapis/python-api-core/issues/234)) ([bdbf889](https://www.github.com/googleapis/python-api-core/commit/bdbf889210b709d7c1945f2160bcba9161b4dd2e)) * strip trailing _ from field mask paths ([#228](https://www.github.com/googleapis/python-api-core/issues/228)) ([ff6ef1b](https://www.github.com/googleapis/python-api-core/commit/ff6ef1bd07fa68307b7c82c910416d770e7b3416)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 11 +++++++++++ google/api_core/version.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8e5e7a5..3629c6e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.0.0](https://www.github.com/googleapis/python-api-core/compare/v2.0.0-b1...v2.0.0) (2021-08-18) + +### ⚠ BREAKING CHANGES + +* drop support for Python 2.7 / 3.5 ([#212](https://www.github.com/googleapis/python-api-core/issues/212)) ([a30f004](https://www.github.com/googleapis/python-api-core/commit/a30f004e74f709d46e905dd819c71f43354e9ac9)) + +### Bug Fixes + +* bump grpcio version to use stable aio API ([#234](https://www.github.com/googleapis/python-api-core/issues/234)) ([bdbf889](https://www.github.com/googleapis/python-api-core/commit/bdbf889210b709d7c1945f2160bcba9161b4dd2e)) +* strip trailing _ from field mask paths ([#228](https://www.github.com/googleapis/python-api-core/issues/228)) ([ff6ef1b](https://www.github.com/googleapis/python-api-core/commit/ff6ef1bd07fa68307b7c82c910416d770e7b3416)) + ## [2.0.0b1](https://www.github.com/googleapis/python-api-core/compare/v1.31.1...v2.0.0b1) (2021-08-03) diff --git a/google/api_core/version.py b/google/api_core/version.py index 5fa77444..a12de3d2 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.0.0b1" +__version__ = "2.0.0" From e3d34d4cf7098f3d4317f28ee3a8f60a3f8c8558 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 15:26:19 +0000 Subject: [PATCH 012/112] chore(python): disable dependency dashboard (#266) --- .github/.OwlBot.lock.yaml | 2 +- renovate.json | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index a9fcd07c..b75186cf 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:9743664022bd63a8084be67f144898314c7ca12f0a03e422ac17c733c129d803 + digest: sha256:d6761eec279244e57fe9d21f8343381a01d3632c034811a72f68b83119e58c69 diff --git a/renovate.json b/renovate.json index c0489556..9fa8816f 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,8 @@ { "extends": [ - "config:base", ":preserveSemverRanges" + "config:base", + ":preserveSemverRanges", + ":disableDependencyDashboard" ], "ignorePaths": [".pre-commit-config.yaml"], "pip_requirements": { From 1db493cafff62e3a9f0b2d8ddf3071199db1af7e Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 30 Aug 2021 13:45:35 -0400 Subject: [PATCH 013/112] chore: migrate default branch to main (#264) --- .kokoro/build.sh | 2 +- .kokoro/test-samples-impl.sh | 2 +- CONTRIBUTING.rst | 12 +++---- docs/conf.py | 10 +++--- google/api_core/exceptions.py | 3 +- google/api_core/future/async_future.py | 6 ++-- google/api_core/future/polling.py | 6 ++-- noxfile.py | 12 ++++++- owlbot.py | 45 ++++++++++++++++++++++++++ 9 files changed, 78 insertions(+), 20 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 8b9fd8a1..0394c8aa 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -41,7 +41,7 @@ python3 -m pip install --upgrade --quiet nox python3 -m nox --version # If this is a continuous build, send the test log to the FlakyBot. -# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. +# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then cleanup() { chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 311a8d54..8a324c9c 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -80,7 +80,7 @@ for file in samples/**/requirements.txt; do EXIT=$? # If this is a periodic build, send the test log to the FlakyBot. - # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. + # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot $KOKORO_GFILE_DIR/linux_amd64/flakybot diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 358404f2..6db668ef 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -49,9 +49,9 @@ You'll have to create a development environment using a Git checkout: # Configure remotes such that you can pull changes from the googleapis/python-api-core # repository into your local repository. $ git remote add upstream git@github.com:googleapis/python-api-core.git - # fetch and merge changes from upstream into master + # fetch and merge changes from upstream into main $ git fetch upstream - $ git merge upstream/master + $ git merge upstream/main Now your local repo is set up such that you will push changes to your GitHub repo, from which you can submit a pull request. @@ -109,12 +109,12 @@ Coding Style variables:: export GOOGLE_CLOUD_TESTING_REMOTE="upstream" - export GOOGLE_CLOUD_TESTING_BRANCH="master" + export GOOGLE_CLOUD_TESTING_BRANCH="main" By doing this, you are specifying the location of the most up-to-date version of ``python-api-core``. The the suggested remote name ``upstream`` should point to the official ``googleapis`` checkout and the - the branch should be the main branch on that remote (``master``). + the branch should be the main branch on that remote (``main``). - This repository contains configuration for the `pre-commit `__ tool, which automates checking @@ -185,7 +185,7 @@ The `description on PyPI`_ for the project comes directly from the ``README``. Due to the reStructuredText (``rst``) parser used by PyPI, relative links which will work on GitHub (e.g. ``CONTRIBUTING.rst`` instead of -``https://github.com/googleapis/python-api-core/blob/master/CONTRIBUTING.rst``) +``https://github.com/googleapis/python-api-core/blob/main/CONTRIBUTING.rst``) may cause problems creating links or rendering the description. .. _description on PyPI: https://pypi.org/project/google-api-core @@ -210,7 +210,7 @@ We support: Supported versions can be found in our ``noxfile.py`` `config`_. -.. _config: https://github.com/googleapis/python-api-core/blob/master/noxfile.py +.. _config: https://github.com/googleapis/python-api-core/blob/main/noxfile.py We also explicitly decided to support Python 3 beginning with version 3.6. diff --git a/docs/conf.py b/docs/conf.py index aec958f3..09f0c2b6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -76,8 +76,8 @@ # The encoding of source files. # source_encoding = 'utf-8-sig' -# The master toctree document. -master_doc = "index" +# The root toctree document. +root_doc = "index" # General information about the project. project = "google-api-core" @@ -280,7 +280,7 @@ # author, documentclass [howto, manual, or own class]). latex_documents = [ ( - master_doc, + root_doc, "google-api-core.tex", "google-api-core Documentation", author, @@ -314,7 +314,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, "google-api-core", "google-api-core Documentation", [author], 1,) + (root_doc, "google-api-core", "google-api-core Documentation", [author], 1,) ] # If true, show URL addresses after external links. @@ -328,7 +328,7 @@ # dir menu entry, description, category) texinfo_documents = [ ( - master_doc, + root_doc, "google-api-core", "google-api-core Documentation", author, diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 13be917e..b0909f1e 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -293,8 +293,7 @@ class Cancelled(ClientError): """Exception mapping a :attr:`grpc.StatusCode.CANCELLED` error.""" # This maps to HTTP status code 499. See - # https://github.com/googleapis/googleapis/blob/master/google/rpc\ - # /code.proto + # https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto code = 499 grpc_status_code = grpc.StatusCode.CANCELLED if grpc is not None else None diff --git a/google/api_core/future/async_future.py b/google/api_core/future/async_future.py index 0343fbe2..88c183f9 100644 --- a/google/api_core/future/async_future.py +++ b/google/api_core/future/async_future.py @@ -43,8 +43,10 @@ class AsyncFuture(base.Future): The :meth:`done` method should be implemented by subclasses. The polling behavior will repeatedly call ``done`` until it returns True. - .. note: Privacy here is intended to prevent the final class from - overexposing, not to prevent subclasses from accessing methods. + .. note:: + + Privacy here is intended to prevent the final class from + overexposing, not to prevent subclasses from accessing methods. Args: retry (google.api_core.retry.Retry): The retry configuration used diff --git a/google/api_core/future/polling.py b/google/api_core/future/polling.py index 2f80efb5..02e680f6 100644 --- a/google/api_core/future/polling.py +++ b/google/api_core/future/polling.py @@ -45,8 +45,10 @@ class PollingFuture(base.Future): The :meth:`done` method should be implemented by subclasses. The polling behavior will repeatedly call ``done`` until it returns True. - .. note: Privacy here is intended to prevent the final class from - overexposing, not to prevent subclasses from accessing methods. + .. note:: + + Privacy here is intended to prevent the final class from + overexposing, not to prevent subclasses from accessing methods. Args: retry (google.api_core.retry.Retry): The retry configuration used diff --git a/noxfile.py b/noxfile.py index 84470f5a..6478bfde 100644 --- a/noxfile.py +++ b/noxfile.py @@ -29,7 +29,17 @@ DEFAULT_PYTHON_VERSION = "3.7" CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() -_MINIMAL_ASYNCIO_SUPPORT_PYTHON_VERSION = [3, 6] +# 'docfx' is excluded since it only needs to run in 'docs-presubmit' +nox.options.sessions = [ + "unit", + "unit_grpc_gcp", + "cover", + "pytype", + "lint", + "lint_setup_py", + "blacken", + "docs", +] def _greater_or_equal_than_36(version_string): diff --git a/owlbot.py b/owlbot.py index 451f7c48..a2f04592 100644 --- a/owlbot.py +++ b/owlbot.py @@ -44,4 +44,49 @@ """, ) +# Remove the replacements below once https://github.com/googleapis/synthtool/pull/1188 is merged + +# Update googleapis/repo-automation-bots repo to main in .kokoro/*.sh files +s.replace( + ".kokoro/*.sh", "repo-automation-bots/tree/master", "repo-automation-bots/tree/main" +) + +# Customize CONTRIBUTING.rst to replace master with main +s.replace( + "CONTRIBUTING.rst", + "fetch and merge changes from upstream into master", + "fetch and merge changes from upstream into main", +) + +s.replace( + "CONTRIBUTING.rst", "git merge upstream/master", "git merge upstream/main", +) + +s.replace( + "CONTRIBUTING.rst", + """export GOOGLE_CLOUD_TESTING_BRANCH=\"master\"""", + """export GOOGLE_CLOUD_TESTING_BRANCH=\"main\"""", +) + +s.replace( + "CONTRIBUTING.rst", r"remote \(``master``\)", "remote (``main``)", +) + +s.replace( + "CONTRIBUTING.rst", "blob/master/CONTRIBUTING.rst", "blob/main/CONTRIBUTING.rst", +) + +s.replace( + "CONTRIBUTING.rst", "blob/master/noxfile.py", "blob/main/noxfile.py", +) + +s.replace( + "docs/conf.py", "master_doc", "root_doc", +) + +s.replace( + "docs/conf.py", "# The master toctree document.", "# The root toctree document.", +) + + s.shell.run(["nox", "-s", "blacken"], hide_output=False) From 82ca2fd3792e20bea20ba96273fde4f2bb07b497 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 31 Aug 2021 13:16:20 -0400 Subject: [PATCH 014/112] tests: close coverage gap for race condition (#261) Closes #260 --- tests/unit/test_bidi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index b876d9ad..15994ee0 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -836,7 +836,7 @@ def test_consumer_unexpected_error(self, caplog): # Wait for the consumer's thread to exit. while consumer.is_active: - pass + pass # pragma: NO COVER (race condition) on_response.assert_not_called() bidi_rpc.recv.assert_called_once() From 618f19201af729205892fcecd9c8e315ba3174a3 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Tue, 31 Aug 2021 12:08:42 -0600 Subject: [PATCH 015/112] fix: do not error on LROs with no response or error (#258) Co-authored-by: Tres Seaver --- google/api_core/operation.py | 10 +++++----- google/api_core/operation_async.py | 10 +++++----- tests/asyncio/test_operation_async.py | 6 +++--- tests/unit/test_operation.py | 6 ++---- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/google/api_core/operation.py b/google/api_core/operation.py index b17f753b..a66e4a50 100644 --- a/google/api_core/operation.py +++ b/google/api_core/operation.py @@ -140,11 +140,11 @@ def _set_result_from_operation(self): ) self.set_exception(exception) else: - exception = exceptions.GoogleAPICallError( - "Unexpected state: Long-running operation had neither " - "response nor error set." - ) - self.set_exception(exception) + # Some APIs set `done: true`, with an empty response. + # Set the result to an empty message of the expected + # result type. + # https://google.aip.dev/151 + self.set_result(self._result_type()) def _refresh_and_update(self, retry=polling.DEFAULT_RETRY): """Refresh the operation and update the result if needed. diff --git a/google/api_core/operation_async.py b/google/api_core/operation_async.py index 6bae8654..17624d62 100644 --- a/google/api_core/operation_async.py +++ b/google/api_core/operation_async.py @@ -136,11 +136,11 @@ def _set_result_from_operation(self): ) self.set_exception(exception) else: - exception = exceptions.GoogleAPICallError( - "Unexpected state: Long-running operation had neither " - "response nor error set." - ) - self.set_exception(exception) + # Some APIs set `done: true`, with an empty response. + # Set the result to an empty message of the expected + # result type. + # https://google.aip.dev/151 + self.set_result(self._result_type()) async def _refresh_and_update(self, retry=async_future.DEFAULT_RETRY): """Refresh the operation and update the result if needed. diff --git a/tests/asyncio/test_operation_async.py b/tests/asyncio/test_operation_async.py index 907cda7c..342184fb 100644 --- a/tests/asyncio/test_operation_async.py +++ b/tests/asyncio/test_operation_async.py @@ -153,7 +153,7 @@ async def test_exception(): @mock.patch("asyncio.sleep", autospec=True) @pytest.mark.asyncio -async def test_unexpected_result(unused_sleep): +async def test_done_with_no_error_or_response(unused_sleep): responses = [ make_operation_proto(), # Second operation response is done, but has not error or response. @@ -161,9 +161,9 @@ async def test_unexpected_result(unused_sleep): ] future, _, _ = make_operation_future(responses) - exception = await future.exception() + result = await future.result() - assert "Unexpected state" in "{!r}".format(exception) + assert isinstance(result, struct_pb2.Struct) def test_from_gapic(): diff --git a/tests/unit/test_operation.py b/tests/unit/test_operation.py index 28fbfe27..7a3e3c6c 100644 --- a/tests/unit/test_operation.py +++ b/tests/unit/test_operation.py @@ -163,7 +163,7 @@ def test_exception_with_error_code(): assert isinstance(exception, exceptions.NotFound) -def test_unexpected_result(): +def test_done_with_no_error_or_response(): responses = [ make_operation_proto(), # Second operation response is done, but has not error or response. @@ -171,9 +171,7 @@ def test_unexpected_result(): ] future, _, _ = make_operation_future(responses) - exception = future.exception() - - assert "Unexpected state" in "{!r}".format(exception) + assert isinstance(future.result(), struct_pb2.Struct) def test__refresh_http(): From dcb6ebd9994fddcb1729150df1675ebf8c503a73 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 31 Aug 2021 15:47:14 -0400 Subject: [PATCH 016/112] chore: release 2.0.1 (#267) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3629c6e9..fdca59ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.0.1](https://www.github.com/googleapis/python-api-core/compare/v2.0.0...v2.0.1) (2021-08-31) + + +### Bug Fixes + +* do not error on LROs with no response or error ([#258](https://www.github.com/googleapis/python-api-core/issues/258)) ([618f192](https://www.github.com/googleapis/python-api-core/commit/618f19201af729205892fcecd9c8e315ba3174a3)) + ## [2.0.0](https://www.github.com/googleapis/python-api-core/compare/v2.0.0-b1...v2.0.0) (2021-08-18) ### ⚠ BREAKING CHANGES diff --git a/google/api_core/version.py b/google/api_core/version.py index a12de3d2..956a957b 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.0.0" +__version__ = "2.0.1" From 01e31ca4ab7b3ca1f60112a02939d62cd27041c7 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 1 Sep 2021 15:44:22 +0000 Subject: [PATCH 017/112] chore(python): group renovate prs (#270) --- .github/.OwlBot.lock.yaml | 2 +- renovate.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index b75186cf..ef3cb34f 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:d6761eec279244e57fe9d21f8343381a01d3632c034811a72f68b83119e58c69 + digest: sha256:1456ea2b3b523ccff5e13030acef56d1de28f21249c62aa0f196265880338fa7 diff --git a/renovate.json b/renovate.json index 9fa8816f..c21036d3 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,7 @@ { "extends": [ "config:base", + "group:all", ":preserveSemverRanges", ":disableDependencyDashboard" ], From 4679219fc1b95bed7f63f6da3b4c0095d359db05 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 1 Sep 2021 12:31:34 -0400 Subject: [PATCH 018/112] chore: remove obsolete dependency in setup.py (#269) Co-authored-by: Tres Seaver --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index cade3d82..48c8979f 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", "setuptools >= 40.3.0", - 'futures >= 3.2.0; python_version < "3.2"', ] extras = { "grpc": "grpcio >= 1.33.2, < 2.0dev", From 8b65c93aef908c99784200b73ad270a0591481a8 Mon Sep 17 00:00:00 2001 From: nicain Date: Thu, 2 Sep 2021 17:50:25 -0700 Subject: [PATCH 019/112] chore: removing owlbot directives for conversion to main (#272) chore: removing owlbot directives for conversion to main --- .github/.OwlBot.lock.yaml | 2 +- owlbot.py | 45 --------------------------------------- 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ef3cb34f..c07f148f 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:1456ea2b3b523ccff5e13030acef56d1de28f21249c62aa0f196265880338fa7 + digest: sha256:0ffe3bdd6c7159692df5f7744da74e5ef19966288a6bf76023e8e04e0c424d7d diff --git a/owlbot.py b/owlbot.py index a2f04592..451f7c48 100644 --- a/owlbot.py +++ b/owlbot.py @@ -44,49 +44,4 @@ """, ) -# Remove the replacements below once https://github.com/googleapis/synthtool/pull/1188 is merged - -# Update googleapis/repo-automation-bots repo to main in .kokoro/*.sh files -s.replace( - ".kokoro/*.sh", "repo-automation-bots/tree/master", "repo-automation-bots/tree/main" -) - -# Customize CONTRIBUTING.rst to replace master with main -s.replace( - "CONTRIBUTING.rst", - "fetch and merge changes from upstream into master", - "fetch and merge changes from upstream into main", -) - -s.replace( - "CONTRIBUTING.rst", "git merge upstream/master", "git merge upstream/main", -) - -s.replace( - "CONTRIBUTING.rst", - """export GOOGLE_CLOUD_TESTING_BRANCH=\"master\"""", - """export GOOGLE_CLOUD_TESTING_BRANCH=\"main\"""", -) - -s.replace( - "CONTRIBUTING.rst", r"remote \(``master``\)", "remote (``main``)", -) - -s.replace( - "CONTRIBUTING.rst", "blob/master/CONTRIBUTING.rst", "blob/main/CONTRIBUTING.rst", -) - -s.replace( - "CONTRIBUTING.rst", "blob/master/noxfile.py", "blob/main/noxfile.py", -) - -s.replace( - "docs/conf.py", "master_doc", "root_doc", -) - -s.replace( - "docs/conf.py", "# The master toctree document.", "# The root toctree document.", -) - - s.shell.run(["nox", "-s", "blacken"], hide_output=False) From afe0fa14c21289c8244606a9f81544cff8ac5f7c Mon Sep 17 00:00:00 2001 From: Yih-Jen Ku Date: Wed, 15 Sep 2021 16:12:18 -0400 Subject: [PATCH 020/112] feat: add grpc transcoding + tests (#259) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add grpc transcoding + tests * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * chore: tweak for clarity / idiomatic usage * chore: attempt to appease Sphinx * feat: add grpc transcoding + tests * Add functions to properly handle subfields * Add unit tests for get_field and delete_field. * Add function docstrings and incorporate correct native dict functions. * Add function docstrings and incorporate correct native dict functions. * Increase code coverage * Increase code coverage * Increase code coverage * Reformat files Co-authored-by: Yonatan Getahun Co-authored-by: Owl Bot Co-authored-by: Tres Seaver --- google/api_core/path_template.py | 107 +++++++++++- tests/unit/test_path_template.py | 274 +++++++++++++++++++++++++++++++ 2 files changed, 380 insertions(+), 1 deletion(-) diff --git a/google/api_core/path_template.py b/google/api_core/path_template.py index c5969c14..41fbd4fe 100644 --- a/google/api_core/path_template.py +++ b/google/api_core/path_template.py @@ -25,6 +25,8 @@ from __future__ import unicode_literals +from collections import deque +import copy import functools import re @@ -64,7 +66,7 @@ def _expand_variable_match(positional_vars, named_vars, match): """Expand a matched variable with its value. Args: - positional_vars (list): A list of positonal variables. This list will + positional_vars (list): A list of positional variables. This list will be modified. named_vars (dict): A dictionary of named variables. match (re.Match): A regular expression match. @@ -170,6 +172,46 @@ def _generate_pattern_for_template(tmpl): return _VARIABLE_RE.sub(_replace_variable_with_pattern, tmpl) +def get_field(request, field): + """Get the value of a field from a given dictionary. + + Args: + request (dict): A dictionary object. + field (str): The key to the request in dot notation. + + Returns: + The value of the field. + """ + parts = field.split(".") + value = request + for part in parts: + if not isinstance(value, dict): + return + value = value.get(part) + if isinstance(value, dict): + return + return value + + +def delete_field(request, field): + """Delete the value of a field from a given dictionary. + + Args: + request (dict): A dictionary object. + field (str): The key to the request in dot notation. + """ + parts = deque(field.split(".")) + while len(parts) > 1: + if not isinstance(request, dict): + return + part = parts.popleft() + request = request.get(part) + part = parts.popleft() + if not isinstance(request, dict): + return + request.pop(part, None) + + def validate(tmpl, path): """Validate a path against the path template. @@ -193,3 +235,66 @@ def validate(tmpl, path): """ pattern = _generate_pattern_for_template(tmpl) + "$" return True if re.match(pattern, path) is not None else False + + +def transcode(http_options, **request_kwargs): + """Transcodes a grpc request pattern into a proper HTTP request following the rules outlined here, + https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44-L312 + + Args: + http_options (list(dict)): A list of dicts which consist of these keys, + 'method' (str): The http method + 'uri' (str): The path template + 'body' (str): The body field name (optional) + (This is a simplified representation of the proto option `google.api.http`) + + request_kwargs (dict) : A dict representing the request object + + Returns: + dict: The transcoded request with these keys, + 'method' (str) : The http method + 'uri' (str) : The expanded uri + 'body' (dict) : A dict representing the body (optional) + 'query_params' (dict) : A dict mapping query parameter variables and values + + Raises: + ValueError: If the request does not match the given template. + """ + for http_option in http_options: + request = {} + + # Assign path + uri_template = http_option["uri"] + path_fields = [ + match.group("name") for match in _VARIABLE_RE.finditer(uri_template) + ] + path_args = {field: get_field(request_kwargs, field) for field in path_fields} + request["uri"] = expand(uri_template, **path_args) + + # Remove fields used in uri path from request + leftovers = copy.deepcopy(request_kwargs) + for path_field in path_fields: + delete_field(leftovers, path_field) + + if not validate(uri_template, request["uri"]) or not all(path_args.values()): + continue + + # Assign body and query params + body = http_option.get("body") + + if body: + if body == "*": + request["body"] = leftovers + request["query_params"] = {} + else: + try: + request["body"] = leftovers.pop(body) + except KeyError: + continue + request["query_params"] = leftovers + else: + request["query_params"] = leftovers + request["method"] = http_option["method"] + return request + + raise ValueError("Request obj does not match any template") diff --git a/tests/unit/test_path_template.py b/tests/unit/test_path_template.py index 4c8a7c5e..2c5216e0 100644 --- a/tests/unit/test_path_template.py +++ b/tests/unit/test_path_template.py @@ -84,6 +84,61 @@ def test_expanded_failure(tmpl, args, kwargs, exc_match): path_template.expand(tmpl, *args, **kwargs) +@pytest.mark.parametrize( + "request_obj, field, expected_result", + [ + [{"field": "stringValue"}, "field", "stringValue"], + [{"field": "stringValue"}, "nosuchfield", None], + [{"field": "stringValue"}, "field.subfield", None], + [{"field": {"subfield": "stringValue"}}, "field", None], + [{"field": {"subfield": "stringValue"}}, "field.subfield", "stringValue"], + [{"field": {"subfield": [1, 2, 3]}}, "field.subfield", [1, 2, 3]], + [{"field": {"subfield": "stringValue"}}, "field", None], + [{"field": {"subfield": "stringValue"}}, "field.nosuchfield", None], + [ + {"field": {"subfield": {"subsubfield": "stringValue"}}}, + "field.subfield.subsubfield", + "stringValue", + ], + ["string", "field", None], + ], +) +def test_get_field(request_obj, field, expected_result): + result = path_template.get_field(request_obj, field) + assert result == expected_result + + +@pytest.mark.parametrize( + "request_obj, field, expected_result", + [ + [{"field": "stringValue"}, "field", {}], + [{"field": "stringValue"}, "nosuchfield", {"field": "stringValue"}], + [{"field": "stringValue"}, "field.subfield", {"field": "stringValue"}], + [{"field": {"subfield": "stringValue"}}, "field.subfield", {"field": {}}], + [ + {"field": {"subfield": "stringValue", "q": "w"}, "e": "f"}, + "field.subfield", + {"field": {"q": "w"}, "e": "f"}, + ], + [ + {"field": {"subfield": "stringValue"}}, + "field.nosuchfield", + {"field": {"subfield": "stringValue"}}, + ], + [ + {"field": {"subfield": {"subsubfield": "stringValue", "q": "w"}}}, + "field.subfield.subsubfield", + {"field": {"subfield": {"q": "w"}}}, + ], + ["string", "field", "string"], + ["string", "field.subfield", "string"], + ], +) +def test_delete_field(request_obj, field, expected_result): + path_template.delete_field(request_obj, field) + assert request_obj == expected_result + + @pytest.mark.parametrize( "tmpl, path", [ @@ -113,3 +168,222 @@ def test__replace_variable_with_pattern(): match.group.return_value = None with pytest.raises(ValueError, match="Unknown"): path_template._replace_variable_with_pattern(match) + + +@pytest.mark.parametrize( + "http_options, request_kwargs, expected_result", + [ + [ + [["get", "/v1/no/template", ""]], + {"foo": "bar"}, + ["get", "/v1/no/template", {}, {"foo": "bar"}], + ], + # Single templates + [ + [["get", "/v1/{field}", ""]], + {"field": "parent"}, + ["get", "/v1/parent", {}, {}], + ], + [ + [["get", "/v1/{field.sub}", ""]], + {"field": {"sub": "parent"}, "foo": "bar"}, + ["get", "/v1/parent", {}, {"field": {}, "foo": "bar"}], + ], + ], +) +def test_transcode_base_case(http_options, request_kwargs, expected_result): + http_options, expected_result = helper_test_transcode(http_options, expected_result) + result = path_template.transcode(http_options, **request_kwargs) + assert result == expected_result + + +@pytest.mark.parametrize( + "http_options, request_kwargs, expected_result", + [ + [ + [["get", "/v1/{field.subfield}", ""]], + {"field": {"subfield": "parent"}, "foo": "bar"}, + ["get", "/v1/parent", {}, {"field": {}, "foo": "bar"}], + ], + [ + [["get", "/v1/{field.subfield.subsubfield}", ""]], + {"field": {"subfield": {"subsubfield": "parent"}}, "foo": "bar"}, + ["get", "/v1/parent", {}, {"field": {"subfield": {}}, "foo": "bar"}], + ], + [ + [["get", "/v1/{field.subfield1}/{field.subfield2}", ""]], + {"field": {"subfield1": "parent", "subfield2": "child"}, "foo": "bar"}, + ["get", "/v1/parent/child", {}, {"field": {}, "foo": "bar"}], + ], + ], +) +def test_transcode_subfields(http_options, request_kwargs, expected_result): + http_options, expected_result = helper_test_transcode(http_options, expected_result) + result = path_template.transcode(http_options, **request_kwargs) + assert result == expected_result + + +@pytest.mark.parametrize( + "http_options, request_kwargs, expected_result", + [ + # Single segment wildcard + [ + [["get", "/v1/{field=*}", ""]], + {"field": "parent"}, + ["get", "/v1/parent", {}, {}], + ], + [ + [["get", "/v1/{field=a/*/b/*}", ""]], + {"field": "a/parent/b/child", "foo": "bar"}, + ["get", "/v1/a/parent/b/child", {}, {"foo": "bar"}], + ], + # Double segment wildcard + [ + [["get", "/v1/{field=**}", ""]], + {"field": "parent/p1"}, + ["get", "/v1/parent/p1", {}, {}], + ], + [ + [["get", "/v1/{field=a/**/b/**}", ""]], + {"field": "a/parent/p1/b/child/c1", "foo": "bar"}, + ["get", "/v1/a/parent/p1/b/child/c1", {}, {"foo": "bar"}], + ], + # Combined single and double segment wildcard + [ + [["get", "/v1/{field=a/*/b/**}", ""]], + {"field": "a/parent/b/child/c1"}, + ["get", "/v1/a/parent/b/child/c1", {}, {}], + ], + [ + [["get", "/v1/{field=a/**/b/*}/v2/{name}", ""]], + {"field": "a/parent/p1/b/child", "name": "first", "foo": "bar"}, + ["get", "/v1/a/parent/p1/b/child/v2/first", {}, {"foo": "bar"}], + ], + ], +) +def test_transcode_with_wildcard(http_options, request_kwargs, expected_result): + http_options, expected_result = helper_test_transcode(http_options, expected_result) + result = path_template.transcode(http_options, **request_kwargs) + assert result == expected_result + + +@pytest.mark.parametrize( + "http_options, request_kwargs, expected_result", + [ + # Single field body + [ + [["post", "/v1/no/template", "data"]], + {"data": {"id": 1, "info": "some info"}, "foo": "bar"}, + ["post", "/v1/no/template", {"id": 1, "info": "some info"}, {"foo": "bar"}], + ], + [ + [["post", "/v1/{field=a/*}/b/{name=**}", "data"]], + { + "field": "a/parent", + "name": "first/last", + "data": {"id": 1, "info": "some info"}, + "foo": "bar", + }, + [ + "post", + "/v1/a/parent/b/first/last", + {"id": 1, "info": "some info"}, + {"foo": "bar"}, + ], + ], + # Wildcard body + [ + [["post", "/v1/{field=a/*}/b/{name=**}", "*"]], + { + "field": "a/parent", + "name": "first/last", + "data": {"id": 1, "info": "some info"}, + "foo": "bar", + }, + [ + "post", + "/v1/a/parent/b/first/last", + {"data": {"id": 1, "info": "some info"}, "foo": "bar"}, + {}, + ], + ], + ], +) +def test_transcode_with_body(http_options, request_kwargs, expected_result): + http_options, expected_result = helper_test_transcode(http_options, expected_result) + result = path_template.transcode(http_options, **request_kwargs) + assert result == expected_result + + +@pytest.mark.parametrize( + "http_options, request_kwargs, expected_result", + [ + # Additional bindings + [ + [ + ["post", "/v1/{field=a/*}/b/{name=**}", "extra_data"], + ["post", "/v1/{field=a/*}/b/{name=**}", "*"], + ], + { + "field": "a/parent", + "name": "first/last", + "data": {"id": 1, "info": "some info"}, + "foo": "bar", + }, + [ + "post", + "/v1/a/parent/b/first/last", + {"data": {"id": 1, "info": "some info"}, "foo": "bar"}, + {}, + ], + ], + [ + [ + ["get", "/v1/{field=a/*}/b/{name=**}", ""], + ["get", "/v1/{field=a/*}/b/first/last", ""], + ], + {"field": "a/parent", "foo": "bar"}, + ["get", "/v1/a/parent/b/first/last", {}, {"foo": "bar"}], + ], + ], +) +def test_transcode_with_additional_bindings( + http_options, request_kwargs, expected_result +): + http_options, expected_result = helper_test_transcode(http_options, expected_result) + result = path_template.transcode(http_options, **request_kwargs) + assert result == expected_result + + +@pytest.mark.parametrize( + "http_options, request_kwargs", + [ + [[["get", "/v1/{name}", ""]], {"foo": "bar"}], + [[["get", "/v1/{name}", ""]], {"name": "first/last"}], + [[["get", "/v1/{name=mr/*/*}", ""]], {"name": "first/last"}], + [[["post", "/v1/{name}", "data"]], {"name": "first/last"}], + ], +) +def test_transcode_fails(http_options, request_kwargs): + http_options, _ = helper_test_transcode(http_options, range(4)) + with pytest.raises(ValueError): + path_template.transcode(http_options, **request_kwargs) + + +def helper_test_transcode(http_options_list, expected_result_list): + http_options = [] + for opt_list in http_options_list: + http_option = {"method": opt_list[0], "uri": opt_list[1]} + if opt_list[2]: + http_option["body"] = opt_list[2] + http_options.append(http_option) + + expected_result = { + "method": expected_result_list[0], + "uri": expected_result_list[1], + "query_params": expected_result_list[3], + } + if expected_result_list[2]: + expected_result["body"] = expected_result_list[2] + + return (http_options, expected_result) From 1c5eb4df93d78e791082d9282330ebf0faacd222 Mon Sep 17 00:00:00 2001 From: Ken Bandes Date: Mon, 20 Sep 2021 14:40:51 -0400 Subject: [PATCH 021/112] feat: Add helper function to format query_params for rest transport. (#275) Co-authored-by: Kenneth Bandes --- google/api_core/rest_helpers.py | 94 +++++++++++++++++++++++++++++++++ tests/unit/test_rest_helpers.py | 77 +++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 google/api_core/rest_helpers.py create mode 100644 tests/unit/test_rest_helpers.py diff --git a/google/api_core/rest_helpers.py b/google/api_core/rest_helpers.py new file mode 100644 index 00000000..23fb614f --- /dev/null +++ b/google/api_core/rest_helpers.py @@ -0,0 +1,94 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for rest transports.""" + +import functools +import operator + + +def flatten_query_params(obj): + """Flatten a nested dict into a list of (name,value) tuples. + + The result is suitable for setting query params on an http request. + + .. code-block:: python + + >>> obj = {'a': + ... {'b': + ... {'c': ['x', 'y', 'z']} }, + ... 'd': 'uvw', } + >>> flatten_query_params(obj) + [('a.b.c', 'x'), ('a.b.c', 'y'), ('a.b.c', 'z'), ('d', 'uvw')] + + Note that, as described in + https://github.com/googleapis/googleapis/blob/48d9fb8c8e287c472af500221c6450ecd45d7d39/google/api/http.proto#L117, + repeated fields (i.e. list-valued fields) may only contain primitive types (not lists or dicts). + This is enforced in this function. + + Args: + obj: a nested dictionary (from json), or None + + Returns: a list of tuples, with each tuple having a (possibly) multi-part name + and a scalar value. + + Raises: + TypeError if obj is not a dict or None + ValueError if obj contains a list of non-primitive values. + """ + + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + + return _flatten(obj, key_path=[]) + + +def _flatten(obj, key_path): + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path) + return _flatten_value(obj, key_path=key_path) + + +def _is_primitive_value(obj): + if obj is None: + return False + + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + + return True + + +def _flatten_value(obj, key_path): + return [(".".join(key_path), obj)] + + +def _flatten_dict(obj, key_path): + items = (_flatten(value, key_path=key_path + [key]) for key, value in obj.items()) + return functools.reduce(operator.concat, items, []) + + +def _flatten_list(elems, key_path): + # Only lists of scalar values are supported. + # The name (key_path) is repeated for each value. + items = ( + _flatten_value(elem, key_path=key_path) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) diff --git a/tests/unit/test_rest_helpers.py b/tests/unit/test_rest_helpers.py new file mode 100644 index 00000000..5932fa55 --- /dev/null +++ b/tests/unit/test_rest_helpers.py @@ -0,0 +1,77 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from google.api_core import rest_helpers + + +def test_flatten_simple_value(): + with pytest.raises(TypeError): + rest_helpers.flatten_query_params("abc") + + +def test_flatten_list(): + with pytest.raises(TypeError): + rest_helpers.flatten_query_params(["abc", "def"]) + + +def test_flatten_none(): + assert rest_helpers.flatten_query_params(None) == [] + + +def test_flatten_empty_dict(): + assert rest_helpers.flatten_query_params({}) == [] + + +def test_flatten_simple_dict(): + assert rest_helpers.flatten_query_params({"a": "abc", "b": "def"}) == [ + ("a", "abc"), + ("b", "def"), + ] + + +def test_flatten_repeated_field(): + assert rest_helpers.flatten_query_params({"a": ["x", "y", "z", None]}) == [ + ("a", "x"), + ("a", "y"), + ("a", "z"), + ] + + +def test_flatten_nested_dict(): + obj = {"a": {"b": {"c": ["x", "y", "z"]}}, "d": {"e": "uvw"}} + expected_result = [("a.b.c", "x"), ("a.b.c", "y"), ("a.b.c", "z"), ("d.e", "uvw")] + + assert rest_helpers.flatten_query_params(obj) == expected_result + + +def test_flatten_repeated_dict(): + obj = { + "a": {"b": {"c": [{"v": 1}, {"v": 2}]}}, + "d": "uvw", + } + + with pytest.raises(ValueError): + rest_helpers.flatten_query_params(obj) + + +def test_flatten_repeated_list(): + obj = { + "a": {"b": {"c": [["e", "f"], ["g", "h"]]}}, + "d": "uvw", + } + + with pytest.raises(ValueError): + rest_helpers.flatten_query_params(obj) From 083e6e92ac870f2ab0bfcbae83fd0f30fe013eb7 Mon Sep 17 00:00:00 2001 From: Jeffrey Rennie Date: Tue, 21 Sep 2021 12:36:22 -0700 Subject: [PATCH 022/112] chore: relocate owl bot post processor (#280) chore: relocate owl bot post processor --- .github/.OwlBot.lock.yaml | 4 ++-- .github/.OwlBot.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index c07f148f..2567653c 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: - image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:0ffe3bdd6c7159692df5f7744da74e5ef19966288a6bf76023e8e04e0c424d7d + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest + digest: sha256:87eee22d276554e4e52863ec9b1cb6a7245815dfae20439712bf644348215a5a diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml index 3caf68d4..c8b40cc7 100644 --- a/.github/.OwlBot.yaml +++ b/.github/.OwlBot.yaml @@ -13,7 +13,7 @@ # limitations under the License. docker: - image: gcr.io/repo-automation-bots/owlbot-python:latest + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest begin-after-commit-hash: 7af2cb8b2b725641ac0d07e2f256d453682802e6 From 3b0912a08f115f352bac65167912400e55ef857e Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 27 Sep 2021 14:45:43 -0400 Subject: [PATCH 023/112] tests: add explicit unit tests for '_StreamedResponseIterator' class (#281) --- tests/unit/test_grpc_helpers.py | 122 ++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py index 12bf1849..f5849290 100644 --- a/tests/unit/test_grpc_helpers.py +++ b/tests/unit/test_grpc_helpers.py @@ -66,6 +66,128 @@ def test_wrap_unary_errors(): assert exc_info.value.response == grpc_error +class Test_StreamingResponseIterator: + @staticmethod + def _make_wrapped(*items): + return iter(items) + + @staticmethod + def _make_one(wrapped, **kw): + return grpc_helpers._StreamingResponseIterator(wrapped, **kw) + + def test_ctor_defaults(self): + wrapped = self._make_wrapped("a", "b", "c") + iterator = self._make_one(wrapped) + assert iterator._stored_first_result == "a" + assert list(wrapped) == ["b", "c"] + + def test_ctor_explicit(self): + wrapped = self._make_wrapped("a", "b", "c") + iterator = self._make_one(wrapped, prefetch_first_result=False) + assert getattr(iterator, "_stored_first_result", self) is self + assert list(wrapped) == ["a", "b", "c"] + + def test_ctor_w_rpc_error_on_prefetch(self): + wrapped = mock.MagicMock() + wrapped.__next__.side_effect = grpc.RpcError() + + with pytest.raises(grpc.RpcError): + self._make_one(wrapped) + + def test___iter__(self): + wrapped = self._make_wrapped("a", "b", "c") + iterator = self._make_one(wrapped) + assert iter(iterator) is iterator + + def test___next___w_cached_first_result(self): + wrapped = self._make_wrapped("a", "b", "c") + iterator = self._make_one(wrapped) + assert next(iterator) == "a" + iterator = self._make_one(wrapped, prefetch_first_result=False) + assert next(iterator) == "b" + assert next(iterator) == "c" + + def test___next___wo_cached_first_result(self): + wrapped = self._make_wrapped("a", "b", "c") + iterator = self._make_one(wrapped, prefetch_first_result=False) + assert next(iterator) == "a" + assert next(iterator) == "b" + assert next(iterator) == "c" + + def test___next___w_rpc_error(self): + wrapped = mock.MagicMock() + wrapped.__next__.side_effect = grpc.RpcError() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + with pytest.raises(exceptions.GoogleAPICallError): + next(iterator) + + def test_add_callback(self): + wrapped = mock.MagicMock() + callback = mock.Mock(spec={}) + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.add_callback(callback) is wrapped.add_callback.return_value + + wrapped.add_callback.assert_called_once_with(callback) + + def test_cancel(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.cancel() is wrapped.cancel.return_value + + wrapped.cancel.assert_called_once_with() + + def test_code(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.code() is wrapped.code.return_value + + wrapped.code.assert_called_once_with() + + def test_details(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.details() is wrapped.details.return_value + + wrapped.details.assert_called_once_with() + + def test_initial_metadata(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.initial_metadata() is wrapped.initial_metadata.return_value + + wrapped.initial_metadata.assert_called_once_with() + + def test_is_active(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.is_active() is wrapped.is_active.return_value + + wrapped.is_active.assert_called_once_with() + + def test_time_remaining(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.time_remaining() is wrapped.time_remaining.return_value + + wrapped.time_remaining.assert_called_once_with() + + def test_trailing_metadata(self): + wrapped = mock.MagicMock() + iterator = self._make_one(wrapped, prefetch_first_result=False) + + assert iterator.trailing_metadata() is wrapped.trailing_metadata.return_value + + wrapped.trailing_metadata.assert_called_once_with() + + def test_wrap_stream_okay(): expected_responses = [1, 2, 3] callable_ = mock.Mock(spec=["__call__"], return_value=iter(expected_responses)) From f3fd05901da03f000b2f3287e7710612653cd6c0 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 4 Oct 2021 11:02:06 -0400 Subject: [PATCH 024/112] chore: add default_version and codeowner_team to .repo-metadata.json (#282) * chore: add default_version and codeowner_team to .repo-metadata.json * Assign @googleapis/actools-python as codeowner --- .repo-metadata.json | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.repo-metadata.json b/.repo-metadata.json index 59aa936d..e16c9d27 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -1,10 +1,12 @@ { - "name": "google-api-core", - "name_pretty": "Google API client core library", - "client_documentation": "https://googleapis.dev/python/google-api-core/latest", - "release_level": "ga", - "language": "python", - "library_type": "CORE", - "repo": "googleapis/python-api-core", - "distribution_name": "google-api-core" -} \ No newline at end of file + "name": "google-api-core", + "name_pretty": "Google API client core library", + "client_documentation": "https://googleapis.dev/python/google-api-core/latest", + "release_level": "ga", + "language": "python", + "library_type": "CORE", + "repo": "googleapis/python-api-core", + "distribution_name": "google-api-core", + "default_version": "", + "codeowner_team": "@googleapis/actools-python" +} From 577da9d35335e4ec3ccb720add22c197444211d9 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 5 Oct 2021 11:07:45 -0600 Subject: [PATCH 025/112] build: use trampoline_v2 for python samples and allow custom dockerfile (#283) Source-Link: https://github.com/googleapis/synthtool/commit/a7ed11ec0863c422ba2e73aafa75eab22c32b33d Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:6e7328583be8edd3ba8f35311c76a1ecbc823010279ccb6ab46b7a76e25eafcc Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/samples/lint/common.cfg | 2 +- .kokoro/samples/python3.6/common.cfg | 2 +- .kokoro/samples/python3.6/periodic.cfg | 2 +- .kokoro/samples/python3.7/common.cfg | 2 +- .kokoro/samples/python3.7/periodic.cfg | 2 +- .kokoro/samples/python3.8/common.cfg | 2 +- .kokoro/samples/python3.8/periodic.cfg | 2 +- .kokoro/samples/python3.9/common.cfg | 2 +- .kokoro/samples/python3.9/periodic.cfg | 2 +- .kokoro/test-samples-against-head.sh | 2 -- .kokoro/test-samples.sh | 2 -- .trampolinerc | 17 ++++++++++++++--- 13 files changed, 24 insertions(+), 17 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 2567653c..ee94722a 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:87eee22d276554e4e52863ec9b1cb6a7245815dfae20439712bf644348215a5a + digest: sha256:6e7328583be8edd3ba8f35311c76a1ecbc823010279ccb6ab46b7a76e25eafcc diff --git a/.kokoro/samples/lint/common.cfg b/.kokoro/samples/lint/common.cfg index f5dddb4b..1a2b87b2 100644 --- a/.kokoro/samples/lint/common.cfg +++ b/.kokoro/samples/lint/common.cfg @@ -31,4 +31,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-api-core/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/common.cfg b/.kokoro/samples/python3.6/common.cfg index 7b4f5cd0..3bb6b3a6 100644 --- a/.kokoro/samples/python3.6/common.cfg +++ b/.kokoro/samples/python3.6/common.cfg @@ -37,4 +37,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-api-core/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/periodic.cfg b/.kokoro/samples/python3.6/periodic.cfg index 50fec964..71cd1e59 100644 --- a/.kokoro/samples/python3.6/periodic.cfg +++ b/.kokoro/samples/python3.6/periodic.cfg @@ -3,4 +3,4 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} diff --git a/.kokoro/samples/python3.7/common.cfg b/.kokoro/samples/python3.7/common.cfg index 1198d7ba..a3aa10b5 100644 --- a/.kokoro/samples/python3.7/common.cfg +++ b/.kokoro/samples/python3.7/common.cfg @@ -37,4 +37,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-api-core/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.7/periodic.cfg b/.kokoro/samples/python3.7/periodic.cfg index 50fec964..71cd1e59 100644 --- a/.kokoro/samples/python3.7/periodic.cfg +++ b/.kokoro/samples/python3.7/periodic.cfg @@ -3,4 +3,4 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} diff --git a/.kokoro/samples/python3.8/common.cfg b/.kokoro/samples/python3.8/common.cfg index b7ec7f5e..20c941aa 100644 --- a/.kokoro/samples/python3.8/common.cfg +++ b/.kokoro/samples/python3.8/common.cfg @@ -37,4 +37,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-api-core/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.8/periodic.cfg b/.kokoro/samples/python3.8/periodic.cfg index 50fec964..71cd1e59 100644 --- a/.kokoro/samples/python3.8/periodic.cfg +++ b/.kokoro/samples/python3.8/periodic.cfg @@ -3,4 +3,4 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} diff --git a/.kokoro/samples/python3.9/common.cfg b/.kokoro/samples/python3.9/common.cfg index cf034ec1..234887c6 100644 --- a/.kokoro/samples/python3.9/common.cfg +++ b/.kokoro/samples/python3.9/common.cfg @@ -37,4 +37,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-api-core/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.9/periodic.cfg b/.kokoro/samples/python3.9/periodic.cfg index 50fec964..71cd1e59 100644 --- a/.kokoro/samples/python3.9/periodic.cfg +++ b/.kokoro/samples/python3.9/periodic.cfg @@ -3,4 +3,4 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} diff --git a/.kokoro/test-samples-against-head.sh b/.kokoro/test-samples-against-head.sh index a7858e4c..ba3a707b 100755 --- a/.kokoro/test-samples-against-head.sh +++ b/.kokoro/test-samples-against-head.sh @@ -23,6 +23,4 @@ set -eo pipefail # Enables `**` to include files nested inside sub-folders shopt -s globstar -cd github/python-api-core - exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh index ee3146bd..11c042d3 100755 --- a/.kokoro/test-samples.sh +++ b/.kokoro/test-samples.sh @@ -24,8 +24,6 @@ set -eo pipefail # Enables `**` to include files nested inside sub-folders shopt -s globstar -cd github/python-api-core - # Run periodic samples tests at latest release if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then # preserving the test runner implementation. diff --git a/.trampolinerc b/.trampolinerc index 383b6ec8..0eee72ab 100644 --- a/.trampolinerc +++ b/.trampolinerc @@ -16,15 +16,26 @@ # Add required env vars here. required_envvars+=( - "STAGING_BUCKET" - "V2_STAGING_BUCKET" ) # Add env vars which are passed down into the container here. pass_down_envvars+=( + "NOX_SESSION" + ############### + # Docs builds + ############### "STAGING_BUCKET" "V2_STAGING_BUCKET" - "NOX_SESSION" + ################## + # Samples builds + ################## + "INSTALL_LIBRARY_FROM_SOURCE" + "RUN_TESTS_SESSION" + "BUILD_SPECIFIC_GCLOUD_PROJECT" + # Target directories. + "RUN_TESTS_DIRS" + # The nox session to run. + "RUN_TESTS_SESSION" ) # Prevent unintentional override on the default image. From a422a5d72cb6f363d57e7a4effe421ba8e049cde Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 5 Oct 2021 16:54:45 -0400 Subject: [PATCH 026/112] feat: add support for Python 3.10 (#284) Fix new deprecation warning for 'threading.Condition.notifyAll'. --- CONTRIBUTING.rst | 6 ++++-- google/api_core/bidi.py | 2 +- noxfile.py | 2 +- setup.py | 1 + 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6db668ef..6b375f03 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -21,7 +21,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.6, 3.7, 3.8 and 3.9 on both UNIX and Windows. + 3.6, 3.7, 3.8, 3.9, and 3.10 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -71,7 +71,7 @@ We use `nox `__ to instrument our tests. - To run a single unit test:: - $ nox -s unit-3.9 -- -k + $ nox -s unit-3.10 -- -k .. note:: @@ -201,11 +201,13 @@ We support: - `Python 3.7`_ - `Python 3.8`_ - `Python 3.9`_ +- `Python 3.10`_ .. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ +.. _Python 3.10: https://docs.python.org/3.10/ Supported versions can be found in our ``noxfile.py`` `config`_. diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index 56a021a9..4b4963f7 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -727,7 +727,7 @@ def resume(self): """Resumes the response stream.""" with self._wake: self._paused = False - self._wake.notifyAll() + self._wake.notify_all() @property def is_paused(self): diff --git a/noxfile.py b/noxfile.py index 6478bfde..617dc580 100644 --- a/noxfile.py +++ b/noxfile.py @@ -121,7 +121,7 @@ def default(session): session.run(*pytest_args) -@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) +@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) def unit(session): """Run the unit test suite.""" default(session) diff --git a/setup.py b/setup.py index 48c8979f..d150bc01 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Operating System :: OS Independent", "Topic :: Internet", ], From 121debdfa3461e0060d3f386b81ca6fa34920234 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 5 Oct 2021 17:08:24 -0400 Subject: [PATCH 027/112] chore: release 2.1.0 (#274) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 9 +++++++++ google/api_core/version.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdca59ba..00910e1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.1.0](https://www.github.com/googleapis/python-api-core/compare/v2.0.1...v2.1.0) (2021-10-05) + + +### Features + +* add grpc transcoding + tests ([#259](https://www.github.com/googleapis/python-api-core/issues/259)) ([afe0fa1](https://www.github.com/googleapis/python-api-core/commit/afe0fa14c21289c8244606a9f81544cff8ac5f7c)) +* Add helper function to format query_params for rest transport. ([#275](https://www.github.com/googleapis/python-api-core/issues/275)) ([1c5eb4d](https://www.github.com/googleapis/python-api-core/commit/1c5eb4df93d78e791082d9282330ebf0faacd222)) +* add support for Python 3.10 ([#284](https://www.github.com/googleapis/python-api-core/issues/284)) ([a422a5d](https://www.github.com/googleapis/python-api-core/commit/a422a5d72cb6f363d57e7a4effe421ba8e049cde)) + ### [2.0.1](https://www.github.com/googleapis/python-api-core/compare/v2.0.0...v2.0.1) (2021-08-31) diff --git a/google/api_core/version.py b/google/api_core/version.py index 956a957b..8b5d3328 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.0.1" +__version__ = "2.1.0" From e6a3489a79da00a5de919a4a41782cc3b1d7f583 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 8 Oct 2021 10:26:00 -0400 Subject: [PATCH 028/112] chore(python): Add kokoro configs for python 3.10 samples testing (#285) Source-Link: https://github.com/googleapis/synthtool/commit/c6e69c4726a233ad8d496961ec265d29e54010b7 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:58f73ba196b5414782605236dd0712a73541b44ff2ff4d3a36ec41092dd6fa5b Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/samples/python3.10/common.cfg | 40 ++++++++++++++++++++ .kokoro/samples/python3.10/continuous.cfg | 6 +++ .kokoro/samples/python3.10/periodic-head.cfg | 11 ++++++ .kokoro/samples/python3.10/periodic.cfg | 6 +++ .kokoro/samples/python3.10/presubmit.cfg | 6 +++ 6 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 .kokoro/samples/python3.10/common.cfg create mode 100644 .kokoro/samples/python3.10/continuous.cfg create mode 100644 .kokoro/samples/python3.10/periodic-head.cfg create mode 100644 .kokoro/samples/python3.10/periodic.cfg create mode 100644 .kokoro/samples/python3.10/presubmit.cfg diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ee94722a..7d98291c 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:6e7328583be8edd3ba8f35311c76a1ecbc823010279ccb6ab46b7a76e25eafcc + digest: sha256:58f73ba196b5414782605236dd0712a73541b44ff2ff4d3a36ec41092dd6fa5b diff --git a/.kokoro/samples/python3.10/common.cfg b/.kokoro/samples/python3.10/common.cfg new file mode 100644 index 00000000..40fb8d81 --- /dev/null +++ b/.kokoro/samples/python3.10/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.10" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-310" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-api-core/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.10/continuous.cfg b/.kokoro/samples/python3.10/continuous.cfg new file mode 100644 index 00000000..a1c8d975 --- /dev/null +++ b/.kokoro/samples/python3.10/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.10/periodic-head.cfg b/.kokoro/samples/python3.10/periodic-head.cfg new file mode 100644 index 00000000..a18c0cfc --- /dev/null +++ b/.kokoro/samples/python3.10/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-api-core/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.10/periodic.cfg b/.kokoro/samples/python3.10/periodic.cfg new file mode 100644 index 00000000..71cd1e59 --- /dev/null +++ b/.kokoro/samples/python3.10/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/.kokoro/samples/python3.10/presubmit.cfg b/.kokoro/samples/python3.10/presubmit.cfg new file mode 100644 index 00000000..a1c8d975 --- /dev/null +++ b/.kokoro/samples/python3.10/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file From 0023ee1fe0e8b80c7a9e8987e0f322a829e5d613 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 13 Oct 2021 17:54:33 -0400 Subject: [PATCH 029/112] fix: add mypy checking + 'py.typed' file (#290) --- google/__init__.py | 2 +- google/api_core/client_info.py | 3 +++ google/api_core/exceptions.py | 9 +++++---- google/api_core/py.typed | 2 ++ mypy.ini | 7 +++++++ noxfile.py | 9 +++++++++ 6 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 google/api_core/py.typed create mode 100644 mypy.ini diff --git a/google/__init__.py b/google/__init__.py index 0d0a4c3a..70a7bd99 100644 --- a/google/__init__.py +++ b/google/__init__.py @@ -21,4 +21,4 @@ except ImportError: import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) + __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore diff --git a/google/api_core/client_info.py b/google/api_core/client_info.py index d7f4367a..e093ffda 100644 --- a/google/api_core/client_info.py +++ b/google/api_core/client_info.py @@ -19,6 +19,7 @@ """ import platform +from typing import Union import pkg_resources @@ -27,6 +28,8 @@ _PY_VERSION = platform.python_version() _API_CORE_VERSION = api_core_version.__version__ +_GRPC_VERSION: Union[str, None] + try: _GRPC_VERSION = pkg_resources.get_distribution("grpcio").version except pkg_resources.DistributionNotFound: # pragma: NO COVER diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index b0909f1e..2cfc2e4a 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -22,17 +22,18 @@ from __future__ import unicode_literals import http.client +from typing import Dict +from typing import Union try: import grpc - except ImportError: # pragma: NO COVER grpc = None # Lookup tables for mapping exceptions from HTTP and gRPC transports. # Populated by _GoogleAPICallErrorMeta -_HTTP_CODE_TO_EXCEPTION = {} -_GRPC_CODE_TO_EXCEPTION = {} +_HTTP_CODE_TO_EXCEPTION: Dict[int, Exception] = {} +_GRPC_CODE_TO_EXCEPTION: Dict[int, Exception] = {} # Additional lookup table to map integer status codes to grpc status code # grpc does not currently support initializing enums from ints @@ -100,7 +101,7 @@ class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): gRPC call metadata. """ - code = None + code: Union[int, None] = None """Optional[int]: The HTTP status code associated with this error. This may be ``None`` if the exception does not have a direct mapping diff --git a/google/api_core/py.typed b/google/api_core/py.typed new file mode 100644 index 00000000..1d5517b1 --- /dev/null +++ b/google/api_core/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google-api-core package uses inline types. diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..5663b40d --- /dev/null +++ b/mypy.ini @@ -0,0 +1,7 @@ +[mypy] +python_version = 3.6 +namespace_packages = True +ignore_missing_imports = True + +[mypy-google.protobuf] +ignore_missing_imports = True diff --git a/noxfile.py b/noxfile.py index 617dc580..926f9f5d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -35,6 +35,7 @@ "unit_grpc_gcp", "cover", "pytype", + "mypy", "lint", "lint_setup_py", "blacken", @@ -155,6 +156,14 @@ def pytype(session): session.run("pytype") +@nox.session(python=DEFAULT_PYTHON_VERSION) +def mypy(session): + """Run type-checking.""" + session.install(".[grpc, grpcgcp]", "mypy") + session.install("types-setuptools", "types-requests", "types-mock") + session.run("mypy", "google", "tests") + + @nox.session(python="3.6") def cover(session): """Run the final coverage report. From f4e776e8889382c28943946e33e29411b02f7436 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 13 Oct 2021 18:09:59 -0400 Subject: [PATCH 030/112] chore: release 2.1.1 (#291) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00910e1f..e8e4e94a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.1.1](https://www.github.com/googleapis/python-api-core/compare/v2.1.0...v2.1.1) (2021-10-13) + + +### Bug Fixes + +* add mypy checking + 'py.typed' file ([#290](https://www.github.com/googleapis/python-api-core/issues/290)) ([0023ee1](https://www.github.com/googleapis/python-api-core/commit/0023ee1fe0e8b80c7a9e8987e0f322a829e5d613)) + ## [2.1.0](https://www.github.com/googleapis/python-api-core/compare/v2.0.1...v2.1.0) (2021-10-05) diff --git a/google/api_core/version.py b/google/api_core/version.py index 8b5d3328..7945f6f4 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.1.0" +__version__ = "2.1.1" From 09cf285536fee519106db335670e1560b0846bcf Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 19 Oct 2021 12:58:59 -0400 Subject: [PATCH 031/112] tests: add testing w/o 'grpc' installed (#289) Closes #288. --- noxfile.py | 14 ++++++++++++-- tests/asyncio/gapic/test_config_async.py | 8 ++++++++ tests/asyncio/gapic/test_method_async.py | 6 +++++- .../operations_v1/test_operations_async_client.py | 10 ++++++++-- tests/asyncio/test_grpc_helpers_async.py | 15 ++++++++++++--- tests/asyncio/test_operation_async.py | 5 +++++ tests/unit/gapic/test_client_info.py | 7 +++++++ tests/unit/gapic/test_config.py | 7 +++++++ tests/unit/gapic/test_method.py | 7 +++++++ tests/unit/gapic/test_routing_header.py | 7 +++++++ .../unit/operations_v1/test_operations_client.py | 7 +++++++ tests/unit/test_bidi.py | 6 +++++- tests/unit/test_client_info.py | 12 +++++++++++- tests/unit/test_exceptions.py | 13 ++++++++++++- tests/unit/test_grpc_helpers.py | 6 +++++- tests/unit/test_operation.py | 6 ++++++ 16 files changed, 124 insertions(+), 12 deletions(-) diff --git a/noxfile.py b/noxfile.py index 926f9f5d..ac1bdd14 100644 --- a/noxfile.py +++ b/noxfile.py @@ -33,6 +33,7 @@ nox.options.sessions = [ "unit", "unit_grpc_gcp", + "unit_wo_grpc", "cover", "pytype", "mypy", @@ -78,7 +79,7 @@ def blacken(session): session.run("black", *BLACK_EXCLUDES, *BLACK_PATHS) -def default(session): +def default(session, install_grpc=True): """Default unit test session. This is intended to be run **without** an interpreter set, so @@ -92,7 +93,10 @@ def default(session): # Install all test dependencies, then install this package in-place. session.install("mock", "pytest", "pytest-cov") - session.install("-e", ".[grpc]", "-c", constraints_path) + if install_grpc: + session.install("-e", ".[grpc]", "-c", constraints_path) + else: + session.install("-e", ".", "-c", constraints_path) pytest_args = [ "python", @@ -140,6 +144,12 @@ def unit_grpc_gcp(session): default(session) +@nox.session(python=["3.6", "3.10"]) +def unit_wo_grpc(session): + """Run the unit test suite w/o grpcio installed""" + default(session, install_grpc=False) + + @nox.session(python="3.6") def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" diff --git a/tests/asyncio/gapic/test_config_async.py b/tests/asyncio/gapic/test_config_async.py index 1f6ea9e2..dbb05d5e 100644 --- a/tests/asyncio/gapic/test_config_async.py +++ b/tests/asyncio/gapic/test_config_async.py @@ -12,6 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. + +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core.gapic_v1 import config_async diff --git a/tests/asyncio/gapic/test_method_async.py b/tests/asyncio/gapic/test_method_async.py index e24fe444..1410747d 100644 --- a/tests/asyncio/gapic/test_method_async.py +++ b/tests/asyncio/gapic/test_method_async.py @@ -14,10 +14,14 @@ import datetime -from grpc import aio import mock import pytest +try: + from grpc import aio +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core import gapic_v1 from google.api_core import grpc_helpers_async diff --git a/tests/asyncio/operations_v1/test_operations_async_client.py b/tests/asyncio/operations_v1/test_operations_async_client.py index 3fb8427b..47c3b4b4 100644 --- a/tests/asyncio/operations_v1/test_operations_async_client.py +++ b/tests/asyncio/operations_v1/test_operations_async_client.py @@ -12,11 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from grpc import aio import mock import pytest -from google.api_core import grpc_helpers_async, operations_v1, page_iterator_async +try: + from grpc import aio +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + +from google.api_core import grpc_helpers_async +from google.api_core import operations_v1 +from google.api_core import page_iterator_async from google.longrunning import operations_pb2 from google.protobuf import empty_pb2 diff --git a/tests/asyncio/test_grpc_helpers_async.py b/tests/asyncio/test_grpc_helpers_async.py index f0b6c5f1..0413abf3 100644 --- a/tests/asyncio/test_grpc_helpers_async.py +++ b/tests/asyncio/test_grpc_helpers_async.py @@ -12,10 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import grpc -from grpc import aio import mock -import pytest +import pytest # noqa: I202 + +try: + import grpc + from grpc import aio +except ImportError: + grpc = aio = None + + +if grpc is None: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core import grpc_helpers_async diff --git a/tests/asyncio/test_operation_async.py b/tests/asyncio/test_operation_async.py index 342184fb..886b1c8d 100644 --- a/tests/asyncio/test_operation_async.py +++ b/tests/asyncio/test_operation_async.py @@ -16,6 +16,11 @@ import mock import pytest +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core import operation_async from google.api_core import operations_v1 diff --git a/tests/unit/gapic/test_client_info.py b/tests/unit/gapic/test_client_info.py index 64080ffd..2ca5c404 100644 --- a/tests/unit/gapic/test_client_info.py +++ b/tests/unit/gapic/test_client_info.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core.gapic_v1 import client_info diff --git a/tests/unit/gapic/test_config.py b/tests/unit/gapic/test_config.py index 1c15261d..5e42fde8 100644 --- a/tests/unit/gapic/test_config.py +++ b/tests/unit/gapic/test_config.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core.gapic_v1 import config diff --git a/tests/unit/gapic/test_method.py b/tests/unit/gapic/test_method.py index e0ea57ac..9778d23a 100644 --- a/tests/unit/gapic/test_method.py +++ b/tests/unit/gapic/test_method.py @@ -15,6 +15,13 @@ import datetime import mock +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core import retry diff --git a/tests/unit/gapic/test_routing_header.py b/tests/unit/gapic/test_routing_header.py index 77300e87..30378676 100644 --- a/tests/unit/gapic/test_routing_header.py +++ b/tests/unit/gapic/test_routing_header.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core.gapic_v1 import routing_header diff --git a/tests/unit/operations_v1/test_operations_client.py b/tests/unit/operations_v1/test_operations_client.py index 001b8fea..187f0be3 100644 --- a/tests/unit/operations_v1/test_operations_client.py +++ b/tests/unit/operations_v1/test_operations_client.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import grpc_helpers from google.api_core import operations_v1 from google.api_core import page_iterator diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index 15994ee0..7fb16209 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -17,10 +17,14 @@ import queue import threading -import grpc import mock import pytest +try: + import grpc +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import bidi from google.api_core import exceptions diff --git a/tests/unit/test_client_info.py b/tests/unit/test_client_info.py index f2274ec2..f5eebfbe 100644 --- a/tests/unit/test_client_info.py +++ b/tests/unit/test_client_info.py @@ -13,6 +13,11 @@ # limitations under the License. +try: + import grpc +except ImportError: + grpc = None + from google.api_core import client_info @@ -20,7 +25,12 @@ def test_constructor_defaults(): info = client_info.ClientInfo() assert info.python_version is not None - assert info.grpc_version is not None + + if grpc is not None: + assert info.grpc_version is not None + else: + assert info.grpc_version is None + assert info.api_core_version is not None assert info.gapic_version is None assert info.client_library_version is None diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index e9709f26..95317dbb 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -15,10 +15,15 @@ import http.client import json -import grpc import mock +import pytest import requests +try: + import grpc +except ImportError: + grpc = None + from google.api_core import exceptions @@ -151,6 +156,7 @@ def test_from_http_response_json_unicode_content(): assert exception.errors == ["1", "2"] +@pytest.mark.skipif(grpc is None, reason="No grpc") def test_from_grpc_status(): message = "message" exception = exceptions.from_grpc_status(grpc.StatusCode.OUT_OF_RANGE, message) @@ -162,6 +168,7 @@ def test_from_grpc_status(): assert exception.errors == [] +@pytest.mark.skipif(grpc is None, reason="No grpc") def test_from_grpc_status_as_int(): message = "message" exception = exceptions.from_grpc_status(11, message) @@ -173,6 +180,7 @@ def test_from_grpc_status_as_int(): assert exception.errors == [] +@pytest.mark.skipif(grpc is None, reason="No grpc") def test_from_grpc_status_with_errors_and_response(): message = "message" response = mock.sentinel.response @@ -187,6 +195,7 @@ def test_from_grpc_status_with_errors_and_response(): assert exception.response == response +@pytest.mark.skipif(grpc is None, reason="No grpc") def test_from_grpc_status_unknown_code(): message = "message" exception = exceptions.from_grpc_status(grpc.StatusCode.OK, message) @@ -194,6 +203,7 @@ def test_from_grpc_status_unknown_code(): assert exception.message == message +@pytest.mark.skipif(grpc is None, reason="No grpc") def test_from_grpc_error(): message = "message" error = mock.create_autospec(grpc.Call, instance=True) @@ -211,6 +221,7 @@ def test_from_grpc_error(): assert exception.response == error +@pytest.mark.skipif(grpc is None, reason="No grpc") def test_from_grpc_error_non_call(): message = "message" error = mock.create_autospec(grpc.RpcError, instance=True) diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py index f5849290..4c613a9b 100644 --- a/tests/unit/test_grpc_helpers.py +++ b/tests/unit/test_grpc_helpers.py @@ -12,10 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import grpc import mock import pytest +try: + import grpc +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) + from google.api_core import exceptions from google.api_core import grpc_helpers import google.auth.credentials diff --git a/tests/unit/test_operation.py b/tests/unit/test_operation.py index 7a3e3c6c..bafff8b0 100644 --- a/tests/unit/test_operation.py +++ b/tests/unit/test_operation.py @@ -14,6 +14,12 @@ import mock +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) from google.api_core import exceptions from google.api_core import operation From ef6f0fcfdfe771172056e35e3c990998b3b00416 Mon Sep 17 00:00:00 2001 From: Aza Tulepbergenov Date: Tue, 19 Oct 2021 11:17:49 -0700 Subject: [PATCH 032/112] feat: add 'GoogleAPICallError.error_details' property (#286) Based on 'google.rpc.status.details'. --- google/api_core/exceptions.py | 67 +++++++++++++++- setup.py | 4 +- testing/constraints-3.6.txt | 3 +- tests/asyncio/test_grpc_helpers_async.py | 3 + tests/unit/test_exceptions.py | 98 ++++++++++++++++++++++-- tests/unit/test_grpc_helpers.py | 3 + 6 files changed, 166 insertions(+), 12 deletions(-) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 2cfc2e4a..fdb21090 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -25,10 +25,14 @@ from typing import Dict from typing import Union +from google.rpc import error_details_pb2 + try: import grpc + from grpc_status import rpc_status except ImportError: # pragma: NO COVER grpc = None + rpc_status = None # Lookup tables for mapping exceptions from HTTP and gRPC transports. # Populated by _GoogleAPICallErrorMeta @@ -97,6 +101,7 @@ class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): Args: message (str): The exception message. errors (Sequence[Any]): An optional list of error details. + details (Sequence[Any]): An optional list of objects defined in google.rpc.error_details. response (Union[requests.Request, grpc.Call]): The response or gRPC call metadata. """ @@ -117,15 +122,19 @@ class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): This may be ``None`` if the exception does not match up to a gRPC error. """ - def __init__(self, message, errors=(), response=None): + def __init__(self, message, errors=(), details=(), response=None): super(GoogleAPICallError, self).__init__(message) self.message = message """str: The exception message.""" self._errors = errors + self._details = details self._response = response def __str__(self): - return "{} {}".format(self.code, self.message) + if self.details: + return "{} {} {}".format(self.code, self.message, self.details) + else: + return "{} {}".format(self.code, self.message) @property def errors(self): @@ -136,6 +145,19 @@ def errors(self): """ return list(self._errors) + @property + def details(self): + """Information contained in google.rpc.status.details. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/status.proto + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto + + Returns: + Sequence[Any]: A list of structured objects from error_details.proto + """ + return list(self._details) + @property def response(self): """Optional[Union[requests.Request, grpc.Call]]: The response or @@ -409,13 +431,15 @@ def from_http_response(response): error_message = payload.get("error", {}).get("message", "unknown error") errors = payload.get("error", {}).get("errors", ()) + # In JSON, details are already formatted in developer-friendly way. + details = payload.get("error", {}).get("details", ()) message = "{method} {url}: {error}".format( method=response.request.method, url=response.request.url, error=error_message ) exception = from_http_status( - response.status_code, message, errors=errors, response=response + response.status_code, message, errors=errors, details=details, response=response ) return exception @@ -462,6 +486,37 @@ def _is_informative_grpc_error(rpc_exc): return hasattr(rpc_exc, "code") and hasattr(rpc_exc, "details") +def _parse_grpc_error_details(rpc_exc): + status = rpc_status.from_call(rpc_exc) + if not status: + return [] + possible_errors = [ + error_details_pb2.BadRequest, + error_details_pb2.PreconditionFailure, + error_details_pb2.QuotaFailure, + error_details_pb2.ErrorInfo, + error_details_pb2.RetryInfo, + error_details_pb2.ResourceInfo, + error_details_pb2.RequestInfo, + error_details_pb2.DebugInfo, + error_details_pb2.Help, + error_details_pb2.LocalizedMessage, + ] + error_details = [] + for detail in status.details: + matched_detail_cls = list( + filter(lambda x: detail.Is(x.DESCRIPTOR), possible_errors) + ) + # If nothing matched, use detail directly. + if len(matched_detail_cls) == 0: + info = detail + else: + info = matched_detail_cls[0]() + detail.Unpack(info) + error_details.append(info) + return error_details + + def from_grpc_error(rpc_exc): """Create a :class:`GoogleAPICallError` from a :class:`grpc.RpcError`. @@ -476,7 +531,11 @@ def from_grpc_error(rpc_exc): # However, check for grpc.RpcError breaks backward compatibility. if isinstance(rpc_exc, grpc.Call) or _is_informative_grpc_error(rpc_exc): return from_grpc_status( - rpc_exc.code(), rpc_exc.details(), errors=(rpc_exc,), response=rpc_exc + rpc_exc.code(), + rpc_exc.details(), + errors=(rpc_exc,), + details=_parse_grpc_error_details(rpc_exc), + response=rpc_exc, ) else: return GoogleAPICallError(str(rpc_exc), errors=(rpc_exc,), response=rpc_exc) diff --git a/setup.py b/setup.py index d150bc01..ddc56004 100644 --- a/setup.py +++ b/setup.py @@ -29,14 +29,14 @@ # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 5 - Production/Stable" dependencies = [ - "googleapis-common-protos >= 1.6.0, < 2.0dev", + "googleapis-common-protos >= 1.52.0, < 2.0dev", "protobuf >= 3.12.0", "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", "setuptools >= 40.3.0", ] extras = { - "grpc": "grpcio >= 1.33.2, < 2.0dev", + "grpc": ["grpcio >= 1.33.2, < 2.0dev", "grpcio-status >= 1.33.2, < 2.0dev"], "grpcgcp": "grpcio-gcp >= 0.2.2", "grpcio-gcp": "grpcio-gcp >= 0.2.2", } diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 744e991b..0c2a07b6 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -5,7 +5,7 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 -googleapis-common-protos==1.6.0 +googleapis-common-protos==1.52.0 protobuf==3.12.0 google-auth==1.25.0 requests==2.18.0 @@ -14,3 +14,4 @@ packaging==14.3 grpcio==1.33.2 grpcio-gcp==0.2.2 grpcio-gcp==0.2.2 +grpcio-status==1.33.2 diff --git a/tests/asyncio/test_grpc_helpers_async.py b/tests/asyncio/test_grpc_helpers_async.py index 0413abf3..3681a40d 100644 --- a/tests/asyncio/test_grpc_helpers_async.py +++ b/tests/asyncio/test_grpc_helpers_async.py @@ -42,6 +42,9 @@ def code(self): def details(self): return None + def trailing_metadata(self): + return None + @pytest.mark.asyncio async def test_wrap_unary_errors(): diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 95317dbb..f6345fe1 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -21,10 +21,13 @@ try: import grpc + from grpc_status import rpc_status except ImportError: - grpc = None + grpc = rpc_status = None from google.api_core import exceptions +from google.protobuf import any_pb2, json_format +from google.rpc import error_details_pb2, status_pb2 def test_create_google_cloud_error(): @@ -38,11 +41,8 @@ def test_create_google_cloud_error(): def test_create_google_cloud_error_with_args(): error = { - "domain": "global", - "location": "test", - "locationType": "testing", + "code": 600, "message": "Testing", - "reason": "test", } response = mock.sentinel.response exception = exceptions.GoogleAPICallError("Testing", [error], response=response) @@ -235,3 +235,91 @@ def test_from_grpc_error_non_call(): assert exception.message == message assert exception.errors == [error] assert exception.response == error + + +def create_bad_request_details(): + bad_request_details = error_details_pb2.BadRequest() + field_violation = bad_request_details.field_violations.add() + field_violation.field = "document.content" + field_violation.description = "Must have some text content to annotate." + status_detail = any_pb2.Any() + status_detail.Pack(bad_request_details) + return status_detail + + +def test_error_details_from_rest_response(): + bad_request_detail = create_bad_request_details() + status = status_pb2.Status() + status.code = 3 + status.message = ( + "3 INVALID_ARGUMENT: One of content, or gcs_content_uri must be set." + ) + status.details.append(bad_request_detail) + + # See JSON schema in https://cloud.google.com/apis/design/errors#http_mapping + http_response = make_response( + json.dumps({"error": json.loads(json_format.MessageToJson(status))}).encode( + "utf-8" + ) + ) + exception = exceptions.from_http_response(http_response) + want_error_details = [json.loads(json_format.MessageToJson(bad_request_detail))] + assert want_error_details == exception.details + # 404 POST comes from make_response. + assert str(exception) == ( + "404 POST https://example.com/: 3 INVALID_ARGUMENT:" + " One of content, or gcs_content_uri must be set." + " [{'@type': 'type.googleapis.com/google.rpc.BadRequest'," + " 'fieldViolations': [{'field': 'document.content'," + " 'description': 'Must have some text content to annotate.'}]}]" + ) + + +def test_error_details_from_v1_rest_response(): + response = make_response( + json.dumps( + {"error": {"message": "\u2019 message", "errors": ["1", "2"]}} + ).encode("utf-8") + ) + exception = exceptions.from_http_response(response) + assert exception.details == [] + + +@pytest.mark.skipif(grpc is None, reason="gRPC not importable") +def test_error_details_from_grpc_response(): + status = rpc_status.status_pb2.Status() + status.code = 3 + status.message = ( + "3 INVALID_ARGUMENT: One of content, or gcs_content_uri must be set." + ) + status_detail = create_bad_request_details() + status.details.append(status_detail) + + # Actualy error doesn't matter as long as its grpc.Call, + # because from_call is mocked. + error = mock.create_autospec(grpc.Call, instance=True) + with mock.patch("grpc_status.rpc_status.from_call") as m: + m.return_value = status + exception = exceptions.from_grpc_error(error) + + bad_request_detail = error_details_pb2.BadRequest() + status_detail.Unpack(bad_request_detail) + assert exception.details == [bad_request_detail] + + +@pytest.mark.skipif(grpc is None, reason="gRPC not importable") +def test_error_details_from_grpc_response_unknown_error(): + status_detail = any_pb2.Any() + + status = rpc_status.status_pb2.Status() + status.code = 3 + status.message = ( + "3 INVALID_ARGUMENT: One of content, or gcs_content_uri must be set." + ) + status.details.append(status_detail) + + error = mock.create_autospec(grpc.Call, instance=True) + with mock.patch("grpc_status.rpc_status.from_call") as m: + m.return_value = status + exception = exceptions.from_grpc_error(error) + assert exception.details == [status_detail] diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py index 4c613a9b..ca969e48 100644 --- a/tests/unit/test_grpc_helpers.py +++ b/tests/unit/test_grpc_helpers.py @@ -56,6 +56,9 @@ def code(self): def details(self): return None + def trailing_metadata(self): + return None + def test_wrap_unary_errors(): grpc_error = RpcErrorImpl(grpc.StatusCode.INVALID_ARGUMENT) From 5f4fb50d2d23b937be81be4f64b54005c5638eb6 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 17:22:41 -0400 Subject: [PATCH 033/112] chore(python): push cloud library docs to staging bucket for Cloud RAD (#295) Source-Link: https://github.com/googleapis/synthtool/commit/7fd61f8efae782a7cfcecc599faf52f9737fe584 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:4ee57a76a176ede9087c14330c625a71553cf9c72828b2c0ca12f5338171ba60 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/docs/common.cfg | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 7d98291c..108063d4 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:58f73ba196b5414782605236dd0712a73541b44ff2ff4d3a36ec41092dd6fa5b + digest: sha256:4ee57a76a176ede9087c14330c625a71553cf9c72828b2c0ca12f5338171ba60 diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg index 4847856f..48e89855 100644 --- a/.kokoro/docs/common.cfg +++ b/.kokoro/docs/common.cfg @@ -30,7 +30,9 @@ env_vars: { env_vars: { key: "V2_STAGING_BUCKET" - value: "docs-staging-v2" + # Push non-cloud library docs to `docs-staging-v2-staging` instead of the + # Cloud RAD bucket `docs-staging-v2` + value: "docs-staging-v2-staging" } # It will upload the docker image after successful builds. From 240edb6a822482865a10ff0bcd5439ccf38d73a3 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 10:09:39 -0600 Subject: [PATCH 034/112] chore: release 2.2.0 (#293) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8e4e94a..f4ea830c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.2.0](https://www.github.com/googleapis/python-api-core/compare/v2.1.1...v2.2.0) (2021-10-25) + + +### Features + +* add 'GoogleAPICallError.error_details' property ([#286](https://www.github.com/googleapis/python-api-core/issues/286)) ([ef6f0fc](https://www.github.com/googleapis/python-api-core/commit/ef6f0fcfdfe771172056e35e3c990998b3b00416)) + ### [2.1.1](https://www.github.com/googleapis/python-api-core/compare/v2.1.0...v2.1.1) (2021-10-13) diff --git a/google/api_core/version.py b/google/api_core/version.py index 7945f6f4..bd0f8e5c 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.1.1" +__version__ = "2.2.0" From 9e6091ee59a30e72a6278b369f6a08e7aef32f22 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Tue, 26 Oct 2021 11:40:20 -0600 Subject: [PATCH 035/112] fix: revert "fix: do not error on LROs with no response or error" (#294) Reverts googleapis/python-api-core#258 From @truchle > A while ago you helped me submit this pull request for the Python LRO client library. This was about making the LRO library not throw an error when receiving an empty LRO response. It turns out that the Python LRO library has always been behaving correctly by accepting empty LRO responses. It would just throw an error if the response field is NULL (see screenshot). > > Since then I've consulted other LRO client library owners (@vam-google ) and it turns out that the fault was in the CCAI Insights server code. CCAI Insights was returning NULL responses instead of empty responses (more context here b/202432905). Since then the issue has been fixed in our server and has rolled out to production, so reverting the Python LRO false fix wouldn't break our code sample. --- google/api_core/operation.py | 10 +++++----- google/api_core/operation_async.py | 10 +++++----- tests/asyncio/test_operation_async.py | 6 +++--- tests/unit/test_operation.py | 6 ++++-- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/google/api_core/operation.py b/google/api_core/operation.py index a66e4a50..b17f753b 100644 --- a/google/api_core/operation.py +++ b/google/api_core/operation.py @@ -140,11 +140,11 @@ def _set_result_from_operation(self): ) self.set_exception(exception) else: - # Some APIs set `done: true`, with an empty response. - # Set the result to an empty message of the expected - # result type. - # https://google.aip.dev/151 - self.set_result(self._result_type()) + exception = exceptions.GoogleAPICallError( + "Unexpected state: Long-running operation had neither " + "response nor error set." + ) + self.set_exception(exception) def _refresh_and_update(self, retry=polling.DEFAULT_RETRY): """Refresh the operation and update the result if needed. diff --git a/google/api_core/operation_async.py b/google/api_core/operation_async.py index 17624d62..6bae8654 100644 --- a/google/api_core/operation_async.py +++ b/google/api_core/operation_async.py @@ -136,11 +136,11 @@ def _set_result_from_operation(self): ) self.set_exception(exception) else: - # Some APIs set `done: true`, with an empty response. - # Set the result to an empty message of the expected - # result type. - # https://google.aip.dev/151 - self.set_result(self._result_type()) + exception = exceptions.GoogleAPICallError( + "Unexpected state: Long-running operation had neither " + "response nor error set." + ) + self.set_exception(exception) async def _refresh_and_update(self, retry=async_future.DEFAULT_RETRY): """Refresh the operation and update the result if needed. diff --git a/tests/asyncio/test_operation_async.py b/tests/asyncio/test_operation_async.py index 886b1c8d..26ad7cef 100644 --- a/tests/asyncio/test_operation_async.py +++ b/tests/asyncio/test_operation_async.py @@ -158,7 +158,7 @@ async def test_exception(): @mock.patch("asyncio.sleep", autospec=True) @pytest.mark.asyncio -async def test_done_with_no_error_or_response(unused_sleep): +async def test_unexpected_result(unused_sleep): responses = [ make_operation_proto(), # Second operation response is done, but has not error or response. @@ -166,9 +166,9 @@ async def test_done_with_no_error_or_response(unused_sleep): ] future, _, _ = make_operation_future(responses) - result = await future.result() + exception = await future.exception() - assert isinstance(result, struct_pb2.Struct) + assert "Unexpected state" in "{!r}".format(exception) def test_from_gapic(): diff --git a/tests/unit/test_operation.py b/tests/unit/test_operation.py index bafff8b0..22e23bc3 100644 --- a/tests/unit/test_operation.py +++ b/tests/unit/test_operation.py @@ -169,7 +169,7 @@ def test_exception_with_error_code(): assert isinstance(exception, exceptions.NotFound) -def test_done_with_no_error_or_response(): +def test_unexpected_result(): responses = [ make_operation_proto(), # Second operation response is done, but has not error or response. @@ -177,7 +177,9 @@ def test_done_with_no_error_or_response(): ] future, _, _ = make_operation_future(responses) - assert isinstance(future.result(), struct_pb2.Struct) + exception = future.exception() + + assert "Unexpected state" in "{!r}".format(exception) def test__refresh_http(): From d2a729e0c42039bfbcd4b59c3d97d8759a9f0d57 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 28 Oct 2021 13:28:08 -0400 Subject: [PATCH 036/112] chore: release 2.2.1 (#296) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4ea830c..8941ad76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.2.1](https://www.github.com/googleapis/python-api-core/compare/v2.2.0...v2.2.1) (2021-10-26) + + +### Bug Fixes + +* revert "fix: do not error on LROs with no response or error" ([#294](https://www.github.com/googleapis/python-api-core/issues/294)) ([9e6091e](https://www.github.com/googleapis/python-api-core/commit/9e6091ee59a30e72a6278b369f6a08e7aef32f22)) + ## [2.2.0](https://www.github.com/googleapis/python-api-core/compare/v2.1.1...v2.2.0) (2021-10-25) diff --git a/google/api_core/version.py b/google/api_core/version.py index bd0f8e5c..b1be7d8c 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.2.0" +__version__ = "2.2.1" From ffc51f03c7ce5d9f009ba859b8df385d52925578 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 2 Nov 2021 18:12:55 -0400 Subject: [PATCH 037/112] fix: make 'gapic_v1.method.DEFAULT' a typed object (#292) FBO checkers which need to verify default values for 'retry' / 'timeout' --- google/api_core/gapic_v1/method.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/google/api_core/gapic_v1/method.py b/google/api_core/gapic_v1/method.py index 79722c07..73c8d4bc 100644 --- a/google/api_core/gapic_v1/method.py +++ b/google/api_core/gapic_v1/method.py @@ -18,6 +18,7 @@ pagination, and long-running operations to gRPC methods. """ +import enum import functools from google.api_core import grpc_helpers @@ -25,7 +26,18 @@ from google.api_core.gapic_v1 import client_info USE_DEFAULT_METADATA = object() -DEFAULT = object() + + +class _MethodDefault(enum.Enum): + # Uses enum so that pytype/mypy knows that this is the only possible value. + # https://stackoverflow.com/a/60605919/101923 + # + # Literal[_DEFAULT_VALUE] is an alternative, but only added in Python 3.8. + # https://docs.python.org/3/library/typing.html#typing.Literal + _DEFAULT_VALUE = object() + + +DEFAULT = _MethodDefault._DEFAULT_VALUE """Sentinel value indicating that a retry or timeout argument was unspecified, so the default should be used.""" From 214d88b54142628f7ecf5d49d3515f353db8aaf5 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 2 Nov 2021 22:16:11 +0000 Subject: [PATCH 038/112] chore: release 2.2.2 (#302) :robot: I have created a release \*beep\* \*boop\* --- ### [2.2.2](https://www.github.com/googleapis/python-api-core/compare/v2.2.1...v2.2.2) (2021-11-02) ### Bug Fixes * make 'gapic_v1.method.DEFAULT' a typed object ([#292](https://www.github.com/googleapis/python-api-core/issues/292)) ([ffc51f0](https://www.github.com/googleapis/python-api-core/commit/ffc51f03c7ce5d9f009ba859b8df385d52925578)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8941ad76..70ce6afc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.2.2](https://www.github.com/googleapis/python-api-core/compare/v2.2.1...v2.2.2) (2021-11-02) + + +### Bug Fixes + +* make 'gapic_v1.method.DEFAULT' a typed object ([#292](https://www.github.com/googleapis/python-api-core/issues/292)) ([ffc51f0](https://www.github.com/googleapis/python-api-core/commit/ffc51f03c7ce5d9f009ba859b8df385d52925578)) + ### [2.2.1](https://www.github.com/googleapis/python-api-core/compare/v2.2.0...v2.2.1) (2021-10-26) diff --git a/google/api_core/version.py b/google/api_core/version.py index b1be7d8c..bcc59c44 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.2.1" +__version__ = "2.2.2" From 060b339e3af296dd1772bfc1b4a0d2b4264cae1f Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 3 Nov 2021 14:18:12 -0400 Subject: [PATCH 039/112] fix: handle bare 'grpc.Call' in 'from_grpc_error' (#298) * fix: handle bare 'grpc.Call' in 'from_grpc_error' Fixes: #297. * tests: add assertion for 'exception.details' --- google/api_core/exceptions.py | 7 ++++++- tests/unit/test_exceptions.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index fdb21090..6b1b6f7e 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -487,9 +487,14 @@ def _is_informative_grpc_error(rpc_exc): def _parse_grpc_error_details(rpc_exc): - status = rpc_status.from_call(rpc_exc) + try: + status = rpc_status.from_call(rpc_exc) + except NotImplementedError: # workaround + return [] + if not status: return [] + possible_errors = [ error_details_pb2.BadRequest, error_details_pb2.PreconditionFailure, diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index f6345fe1..622f58ab 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -237,6 +237,34 @@ def test_from_grpc_error_non_call(): assert exception.response == error +@pytest.mark.skipif(grpc is None, reason="No grpc") +def test_from_grpc_error_bare_call(): + message = "Testing" + + class TestingError(grpc.Call, grpc.RpcError): + def __init__(self, exception): + self.exception = exception + + def code(self): + return self.exception.grpc_status_code + + def details(self): + return message + + nested_message = "message" + error = TestingError(exceptions.GoogleAPICallError(nested_message)) + + exception = exceptions.from_grpc_error(error) + + assert isinstance(exception, exceptions.GoogleAPICallError) + assert exception.code is None + assert exception.grpc_status_code is None + assert exception.message == message + assert exception.errors == [error] + assert exception.response == error + assert exception.details == [] + + def create_bad_request_details(): bad_request_details = error_details_pb2.BadRequest() field_violation = bad_request_details.field_violations.add() From 4b407899cdcaaada0e1b9e12fbf8213690402ce5 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 13:12:09 -0500 Subject: [PATCH 040/112] chore(python): add .github/CODEOWNERS as a templated file (#308) Source-Link: https://github.com/googleapis/synthtool/commit/c5026b3217973a8db55db8ee85feee0e9a65e295 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:0e18b9475fbeb12d9ad4302283171edebb6baf2dfca1bd215ee3b34ed79d95d7 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- .github/CODEOWNERS | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 108063d4..7519fa3a 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:4ee57a76a176ede9087c14330c625a71553cf9c72828b2c0ca12f5338171ba60 + digest: sha256:0e18b9475fbeb12d9ad4302283171edebb6baf2dfca1bd215ee3b34ed79d95d7 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 557e39e1..ee818917 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,9 +3,10 @@ # # For syntax help see: # https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax +# Note: This file is autogenerated. To make changes to the codeowner team, please update .repo-metadata.json. -# The @googleapis/yoshi-python is the default owner for changes in this repo -* @googleapis/yoshi-python @googleapis/actools-python +# @googleapis/yoshi-python @googleapis/actools-python are the default owners for changes in this repo +* @googleapis/yoshi-python @googleapis/actools-python -# The python-samples-reviewers team is the default owner for samples changes -/samples/ @googleapis/python-samples-owners +# @googleapis/python-samples-owners @googleapis/actools-python are the default owners for samples changes +/samples/ @googleapis/python-samples-owners @googleapis/actools-python From bc0abe48e1270d58245d04ba0bd5f878fa945c69 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 18 Nov 2021 14:58:20 -0500 Subject: [PATCH 041/112] ci: tweak mypy to check 'google.protobuf' (#310) Install the 'types-protobuf' package in support. --- google/__init__.py | 1 + mypy.ini | 3 --- noxfile.py | 4 +++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/google/__init__.py b/google/__init__.py index 70a7bd99..9f1d5491 100644 --- a/google/__init__.py +++ b/google/__init__.py @@ -21,4 +21,5 @@ except ImportError: import pkgutil + # See: https://github.com/python/mypy/issues/1422 __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore diff --git a/mypy.ini b/mypy.ini index 5663b40d..5c111571 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,6 +2,3 @@ python_version = 3.6 namespace_packages = True ignore_missing_imports = True - -[mypy-google.protobuf] -ignore_missing_imports = True diff --git a/noxfile.py b/noxfile.py index ac1bdd14..db37c561 100644 --- a/noxfile.py +++ b/noxfile.py @@ -170,7 +170,9 @@ def pytype(session): def mypy(session): """Run type-checking.""" session.install(".[grpc, grpcgcp]", "mypy") - session.install("types-setuptools", "types-requests", "types-mock") + session.install( + "types-setuptools", "types-requests", "types-protobuf", "types-mock" + ) session.run("mypy", "google", "tests") From ce1adf395982ede157c0f25a920946bb52789873 Mon Sep 17 00:00:00 2001 From: Ken Bandes Date: Wed, 24 Nov 2021 21:17:42 -0500 Subject: [PATCH 042/112] feat: add operations rest client to support long-running operations. (#311) * feat: add operations rest client to support long-running operations. * fix: address test coverage gaps in operations rest client. * fix: removed stray print statement. * fix: address lint, blacken, and mypy issues. * fix: address pytype, more coverage issues * fix: addressed additional pytype issues and one coverage line. * fix: renamed OperationsRestClient to AbstractOperationsClient. Co-authored-by: Kenneth Bandes --- google/api_core/operations_v1/__init__.py | 9 +- .../abstract_operations_client.py | 564 +++++++++++ google/api_core/operations_v1/pagers.py | 86 ++ .../operations_v1/transports/__init__.py | 30 + .../api_core/operations_v1/transports/base.py | 232 +++++ .../api_core/operations_v1/transports/rest.py | 455 +++++++++ .../test_operations_rest_client.py | 944 ++++++++++++++++++ 7 files changed, 2319 insertions(+), 1 deletion(-) create mode 100644 google/api_core/operations_v1/abstract_operations_client.py create mode 100644 google/api_core/operations_v1/pagers.py create mode 100644 google/api_core/operations_v1/transports/__init__.py create mode 100644 google/api_core/operations_v1/transports/base.py create mode 100644 google/api_core/operations_v1/transports/rest.py create mode 100644 tests/unit/operations_v1/test_operations_rest_client.py diff --git a/google/api_core/operations_v1/__init__.py b/google/api_core/operations_v1/__init__.py index d7f963e2..61186451 100644 --- a/google/api_core/operations_v1/__init__.py +++ b/google/api_core/operations_v1/__init__.py @@ -14,7 +14,14 @@ """Package for interacting with the google.longrunning.operations meta-API.""" +from google.api_core.operations_v1.abstract_operations_client import AbstractOperationsClient from google.api_core.operations_v1.operations_async_client import OperationsAsyncClient from google.api_core.operations_v1.operations_client import OperationsClient +from google.api_core.operations_v1.transports.rest import OperationsRestTransport -__all__ = ["OperationsAsyncClient", "OperationsClient"] +__all__ = [ + "AbstractOperationsClient", + "OperationsAsyncClient", + "OperationsClient", + "OperationsRestTransport" +] diff --git a/google/api_core/operations_v1/abstract_operations_client.py b/google/api_core/operations_v1/abstract_operations_client.py new file mode 100644 index 00000000..631094e7 --- /dev/null +++ b/google/api_core/operations_v1/abstract_operations_client.py @@ -0,0 +1,564 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from distutils import util +import os +import re +from typing import Dict, Optional, Sequence, Tuple, Type, Union + +from google.api_core import client_options as client_options_lib # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.api_core.operations_v1 import pagers +from google.api_core.operations_v1.transports.base import ( + DEFAULT_CLIENT_INFO, + OperationsTransport, +) +from google.api_core.operations_v1.transports.rest import OperationsRestTransport +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.exceptions import MutualTLSChannelError # type: ignore +from google.auth.transport import mtls # type: ignore +from google.longrunning import operations_pb2 +from google.oauth2 import service_account # type: ignore + +OptionalRetry = Union[retries.Retry, object] + + +class AbstractOperationsClientMeta(type): + """Metaclass for the Operations client. + + This provides class-level methods for building and retrieving + support objects (e.g. transport) without polluting the client instance + objects. + """ + + _transport_registry = OrderedDict() # type: Dict[str, Type[OperationsTransport]] + _transport_registry["rest"] = OperationsRestTransport + + def get_transport_class( + cls, label: Optional[str] = None, + ) -> Type[OperationsTransport]: + """Returns an appropriate transport class. + + Args: + label: The name of the desired transport. If none is + provided, then the first transport in the registry is used. + + Returns: + The transport class to use. + """ + # If a specific transport is requested, return that one. + if label: + return cls._transport_registry[label] + + # No transport is requested; return the default (that is, the first one + # in the dictionary). + return next(iter(cls._transport_registry.values())) + + +class AbstractOperationsClient(metaclass=AbstractOperationsClientMeta): + """Manages long-running operations with an API service. + + When an API method normally takes long time to complete, it can be + designed to return [Operation][google.api_core.operations_v1.Operation] to the + client, and the client can use this interface to receive the real + response asynchronously by polling the operation resource, or pass + the operation resource to another API (such as Google Cloud Pub/Sub + API) to receive the response. Any API service that returns + long-running operations should implement the ``Operations`` + interface so developers can have a consistent client experience. + """ + + @staticmethod + def _get_default_mtls_endpoint(api_endpoint): + """Converts api endpoint to mTLS endpoint. + + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to + "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. + Args: + api_endpoint (Optional[str]): the api endpoint to convert. + Returns: + str: converted mTLS api endpoint. + """ + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + DEFAULT_ENDPOINT = "longrunning.googleapis.com" + DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore + DEFAULT_ENDPOINT + ) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + AbstractOperationsClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + AbstractOperationsClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file(filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> OperationsTransport: + """Returns the transport used by the client instance. + + Returns: + OperationsTransport: The transport used by the client + instance. + """ + return self._transport + + @staticmethod + def common_billing_account_path(billing_account: str,) -> str: + """Returns a fully-qualified billing_account string.""" + return "billingAccounts/{billing_account}".format( + billing_account=billing_account, + ) + + @staticmethod + def parse_common_billing_account_path(path: str) -> Dict[str, str]: + """Parse a billing_account path into its component segments.""" + m = re.match(r"^billingAccounts/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_folder_path(folder: str,) -> str: + """Returns a fully-qualified folder string.""" + return "folders/{folder}".format(folder=folder,) + + @staticmethod + def parse_common_folder_path(path: str) -> Dict[str, str]: + """Parse a folder path into its component segments.""" + m = re.match(r"^folders/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_organization_path(organization: str,) -> str: + """Returns a fully-qualified organization string.""" + return "organizations/{organization}".format(organization=organization,) + + @staticmethod + def parse_common_organization_path(path: str) -> Dict[str, str]: + """Parse a organization path into its component segments.""" + m = re.match(r"^organizations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_project_path(project: str,) -> str: + """Returns a fully-qualified project string.""" + return "projects/{project}".format(project=project,) + + @staticmethod + def parse_common_project_path(path: str) -> Dict[str, str]: + """Parse a project path into its component segments.""" + m = re.match(r"^projects/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_location_path(project: str, location: str,) -> str: + """Returns a fully-qualified location string.""" + return "projects/{project}/locations/{location}".format( + project=project, location=location, + ) + + @staticmethod + def parse_common_location_path(path: str) -> Dict[str, str]: + """Parse a location path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) + return m.groupdict() if m else {} + + def __init__( + self, + *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Union[str, OperationsTransport, None] = None, + client_options: Optional[client_options_lib.ClientOptions] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the operations client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Union[str, OperationsTransport]): The + transport to use. If set to None, a transport is chosen + automatically. + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. It won't take effect if a ``transport`` instance is provided. + (1) The ``api_endpoint`` property can be used to override the + default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT + environment variable can also be used to override the endpoint: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto switch to the + default mTLS endpoint if client certificate is present, this is + the default value). However, the ``api_endpoint`` property takes + precedence if provided. + (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide client certificate for mutual TLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + """ + if isinstance(client_options, dict): + client_options = client_options_lib.from_dict(client_options) + if client_options is None: + client_options = client_options_lib.ClientOptions() + + # Create SSL credentials for mutual TLS if needed. + use_client_cert = bool( + util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false")) + ) + + client_cert_source_func = None + is_mtls = False + if use_client_cert: + if client_options.client_cert_source: + is_mtls = True + client_cert_source_func = client_options.client_cert_source + else: + is_mtls = mtls.has_default_client_cert_source() + if is_mtls: + client_cert_source_func = mtls.default_client_cert_source() + else: + client_cert_source_func = None + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + else: + use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_mtls_env == "never": + api_endpoint = self.DEFAULT_ENDPOINT + elif use_mtls_env == "always": + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + elif use_mtls_env == "auto": + if is_mtls: + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = self.DEFAULT_ENDPOINT + else: + raise MutualTLSChannelError( + "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " + "values: never, auto, always" + ) + + # Save or instantiate the transport. + # Ordinarily, we provide the transport, but allowing a custom transport + # instance provides an extensibility point for unusual situations. + if isinstance(transport, OperationsTransport): + # transport is a OperationsTransport instance. + if credentials or client_options.credentials_file: + raise ValueError( + "When providing a transport instance, " + "provide its credentials directly." + ) + if client_options.scopes: + raise ValueError( + "When providing a transport instance, provide its scopes " + "directly." + ) + self._transport = transport + else: + Transport = type(self).get_transport_class(transport) + self._transport = Transport( + credentials=credentials, + credentials_file=client_options.credentials_file, + host=api_endpoint, + scopes=client_options.scopes, + client_cert_source_for_mtls=client_cert_source_func, + quota_project_id=client_options.quota_project_id, + client_info=client_info, + always_use_jwt_access=True, + ) + + def list_operations( + self, + name: str, + filter_: Optional[str] = None, + *, + page_size: Optional[int] = None, + page_token: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> pagers.ListOperationsPager: + r"""Lists operations that match the specified filter in the request. + If the server doesn't support this method, it returns + ``UNIMPLEMENTED``. + + NOTE: the ``name`` binding allows API services to override the + binding to use different resource name schemes, such as + ``users/*/operations``. To override the binding, API services + can add a binding such as ``"/v1/{name=users/*}/operations"`` to + their service configuration. For backwards compatibility, the + default name includes the operations collection id, however + overriding users must ensure the name binding is the parent + resource, without the operations collection id. + + Args: + name (str): + The name of the operation's parent + resource. + filter_ (str): + The standard list filter. + This corresponds to the ``filter`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.api_core.operations_v1.pagers.ListOperationsPager: + The response message for + [Operations.ListOperations][google.api_core.operations_v1.Operations.ListOperations]. + + Iterating over this object will yield results and + resolve additional pages automatically. + + """ + # Create a protobuf request object. + request = operations_pb2.ListOperationsRequest(name=name, filter=filter_) + if page_size is not None: + request.page_size = page_size + if page_token is not None: + request.page_token = page_token + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.list_operations] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata or ()) + ( + gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), + ) + + # Send the request. + response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # This method is paged; wrap the response in a pager, which provides + # an `__iter__` convenience method. + response = pagers.ListOperationsPager( + method=rpc, request=request, response=response, metadata=metadata, + ) + + # Done; return the response. + return response + + def get_operation( + self, + name: str, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> operations_pb2.Operation: + r"""Gets the latest state of a long-running operation. + Clients can use this method to poll the operation result + at intervals as recommended by the API service. + + Args: + name (str): + The name of the operation resource. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.longrunning.operations_pb2.Operation: + This resource represents a long- + unning operation that is the result of a + network API call. + + """ + + request = operations_pb2.GetOperationRequest(name=name) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.get_operation] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata or ()) + ( + gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), + ) + + # Send the request. + response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + def delete_operation( + self, + name: str, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> None: + r"""Deletes a long-running operation. This method indicates that the + client is no longer interested in the operation result. It does + not cancel the operation. If the server doesn't support this + method, it returns ``google.rpc.Code.UNIMPLEMENTED``. + + Args: + name (str): + The name of the operation resource to + be deleted. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + """ + # Create the request object. + request = operations_pb2.DeleteOperationRequest(name=name) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.delete_operation] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata or ()) + ( + gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), + ) + + # Send the request. + rpc( + request, retry=retry, timeout=timeout, metadata=metadata, + ) + + def cancel_operation( + self, + name: Optional[str] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> None: + r"""Starts asynchronous cancellation on a long-running operation. + The server makes a best effort to cancel the operation, but + success is not guaranteed. If the server doesn't support this + method, it returns ``google.rpc.Code.UNIMPLEMENTED``. Clients + can use + [Operations.GetOperation][google.api_core.operations_v1.Operations.GetOperation] + or other methods to check whether the cancellation succeeded or + whether the operation completed despite cancellation. On + successful cancellation, the operation is not deleted; instead, + it becomes an operation with an + [Operation.error][google.api_core.operations_v1.Operation.error] value with + a [google.rpc.Status.code][google.rpc.Status.code] of 1, + corresponding to ``Code.CANCELLED``. + + Args: + name (str): + The name of the operation resource to + be cancelled. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + """ + # Create the request object. + request = operations_pb2.CancelOperationRequest(name=name) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.cancel_operation] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata or ()) + ( + gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), + ) + + # Send the request. + rpc( + request, retry=retry, timeout=timeout, metadata=metadata, + ) diff --git a/google/api_core/operations_v1/pagers.py b/google/api_core/operations_v1/pagers.py new file mode 100644 index 00000000..b8a47757 --- /dev/null +++ b/google/api_core/operations_v1/pagers.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import ( + Any, + Callable, + Iterator, + Sequence, + Tuple, +) + +from google.longrunning import operations_pb2 + + +class ListOperationsPager: + """A pager for iterating through ``list_operations`` requests. + + This class thinly wraps an initial + :class:`google.longrunning.operations_pb2.ListOperationsResponse` object, and + provides an ``__iter__`` method to iterate through its + ``operations`` field. + + If there are more pages, the ``__iter__`` method will make additional + ``ListOperations`` requests and continue to iterate + through the ``operations`` field on the + corresponding responses. + + All the usual :class:`google.longrunning.operations_pb2.ListOperationsResponse` + attributes are available on the pager. If multiple requests are made, only + the most recent response is retained, and thus used for attribute lookup. + """ + + def __init__( + self, + method: Callable[..., operations_pb2.ListOperationsResponse], + request: operations_pb2.ListOperationsRequest, + response: operations_pb2.ListOperationsResponse, + *, + metadata: Sequence[Tuple[str, str]] = () + ): + """Instantiate the pager. + + Args: + method (Callable): The method that was originally called, and + which instantiated this pager. + request (google.longrunning.operations_pb2.ListOperationsRequest): + The initial request object. + response (google.longrunning.operations_pb2.ListOperationsResponse): + The initial response object. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + """ + self._method = method + self._request = request + self._response = response + self._metadata = metadata + + def __getattr__(self, name: str) -> Any: + return getattr(self._response, name) + + @property + def pages(self) -> Iterator[operations_pb2.ListOperationsResponse]: + yield self._response + while self._response.next_page_token: + self._request.page_token = self._response.next_page_token + self._response = self._method(self._request, metadata=self._metadata) + yield self._response + + def __iter__(self) -> Iterator[operations_pb2.Operation]: + for page in self.pages: + yield from page.operations + + def __repr__(self) -> str: + return "{0}<{1!r}>".format(self.__class__.__name__, self._response) diff --git a/google/api_core/operations_v1/transports/__init__.py b/google/api_core/operations_v1/transports/__init__.py new file mode 100644 index 00000000..b443c078 --- /dev/null +++ b/google/api_core/operations_v1/transports/__init__.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from typing import Dict, Type + +from .base import OperationsTransport +from .rest import OperationsRestTransport + + +# Compile a registry of transports. +_transport_registry = OrderedDict() # type: Dict[str, Type[OperationsTransport]] +_transport_registry["rest"] = OperationsRestTransport + +__all__ = ( + "OperationsTransport", + "OperationsRestTransport", +) diff --git a/google/api_core/operations_v1/transports/base.py b/google/api_core/operations_v1/transports/base.py new file mode 100644 index 00000000..460e6460 --- /dev/null +++ b/google/api_core/operations_v1/transports/base.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +from typing import Awaitable, Callable, Optional, Sequence, Union + +import pkg_resources + +import google.api_core # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.longrunning import operations_pb2 +from google.oauth2 import service_account # type: ignore +from google.protobuf import empty_pb2 # type: ignore + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + "google.api_core.operations_v1", + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + + +class OperationsTransport(abc.ABC): + """Abstract transport class for Operations.""" + + AUTH_SCOPES = () + + DEFAULT_HOST: str = "longrunning.googleapis.com" + + def __init__( + self, + *, + host: str = DEFAULT_HOST, + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + **kwargs, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A list of scopes. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + """ + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ":" not in host: + host += ":443" + self._host = host + + scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES} + + # Save the scopes. + self._scopes = scopes + + # If no credentials are provided, then determine the appropriate + # defaults. + if credentials and credentials_file: + raise core_exceptions.DuplicateCredentialArgs( + "'credentials_file' and 'credentials' are mutually exclusive" + ) + + if credentials_file is not None: + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, **scopes_kwargs, quota_project_id=quota_project_id + ) + + elif credentials is None: + credentials, _ = google.auth.default( + **scopes_kwargs, quota_project_id=quota_project_id + ) + + # If the credentials are service account credentials, then always try to use self signed JWT. + if ( + always_use_jwt_access + and isinstance(credentials, service_account.Credentials) + and hasattr(service_account.Credentials, "with_always_use_jwt_access") + ): + credentials = credentials.with_always_use_jwt_access(True) + + # Save the credentials. + self._credentials = credentials + + def _prep_wrapped_messages(self, client_info): + # Precompute the wrapped methods. + self._wrapped_methods = { + self.list_operations: gapic_v1.method.wrap_method( + self.list_operations, + default_retry=retries.Retry( + initial=0.5, + maximum=10.0, + multiplier=2.0, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=10.0, + ), + default_timeout=10.0, + client_info=client_info, + ), + self.get_operation: gapic_v1.method.wrap_method( + self.get_operation, + default_retry=retries.Retry( + initial=0.5, + maximum=10.0, + multiplier=2.0, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=10.0, + ), + default_timeout=10.0, + client_info=client_info, + ), + self.delete_operation: gapic_v1.method.wrap_method( + self.delete_operation, + default_retry=retries.Retry( + initial=0.5, + maximum=10.0, + multiplier=2.0, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=10.0, + ), + default_timeout=10.0, + client_info=client_info, + ), + self.cancel_operation: gapic_v1.method.wrap_method( + self.cancel_operation, + default_retry=retries.Retry( + initial=0.5, + maximum=10.0, + multiplier=2.0, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=10.0, + ), + default_timeout=10.0, + client_info=client_info, + ), + } + + def close(self): + """Closes resources associated with the transport. + + .. warning:: + Only call this method if the transport is NOT shared + with other clients - this may cause errors in other clients! + """ + raise NotImplementedError() + + @property + def list_operations( + self, + ) -> Callable[ + [operations_pb2.ListOperationsRequest], + Union[ + operations_pb2.ListOperationsResponse, + Awaitable[operations_pb2.ListOperationsResponse], + ], + ]: + raise NotImplementedError() + + @property + def get_operation( + self, + ) -> Callable[ + [operations_pb2.GetOperationRequest], + Union[operations_pb2.Operation, Awaitable[operations_pb2.Operation]], + ]: + raise NotImplementedError() + + @property + def delete_operation( + self, + ) -> Callable[ + [operations_pb2.DeleteOperationRequest], + Union[empty_pb2.Empty, Awaitable[empty_pb2.Empty]], + ]: + raise NotImplementedError() + + @property + def cancel_operation( + self, + ) -> Callable[ + [operations_pb2.CancelOperationRequest], + Union[empty_pb2.Empty, Awaitable[empty_pb2.Empty]], + ]: + raise NotImplementedError() + + +__all__ = ("OperationsTransport",) diff --git a/google/api_core/operations_v1/transports/rest.py b/google/api_core/operations_v1/transports/rest.py new file mode 100644 index 00000000..27ed7661 --- /dev/null +++ b/google/api_core/operations_v1/transports/rest.py @@ -0,0 +1,455 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Callable, Dict, Optional, Sequence, Tuple, Union + +from requests import __version__ as requests_version + +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import path_template # type: ignore +from google.api_core import rest_helpers # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.requests import AuthorizedSession # type: ignore +from google.longrunning import operations_pb2 # type: ignore +from google.protobuf import empty_pb2 # type: ignore +from google.protobuf import json_format # type: ignore +from .base import DEFAULT_CLIENT_INFO as BASE_DEFAULT_CLIENT_INFO, OperationsTransport + +OptionalRetry = Union[retries.Retry, object] + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=BASE_DEFAULT_CLIENT_INFO.gapic_version, + grpc_version=None, + rest_version=requests_version, +) + + +class OperationsRestTransport(OperationsTransport): + """REST backend transport for Operations. + + Manages long-running operations with an API service. + + When an API method normally takes long time to complete, it can be + designed to return [Operation][google.api_core.operations_v1.Operation] to the + client, and the client can use this interface to receive the real + response asynchronously by polling the operation resource, or pass + the operation resource to another API (such as Google Cloud Pub/Sub + API) to receive the response. Any API service that returns + long-running operations should implement the ``Operations`` + interface so developers can have a consistent client experience. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends JSON representations of protocol buffers over HTTP/1.1 + """ + + def __init__( + self, + *, + host: str = "longrunning.googleapis.com", + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + url_scheme: str = "https", + http_options: Optional[Dict] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if ``channel`` is provided. + client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client + certificate to configure mutual TLS HTTP channel. It is ignored + if ``channel`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + url_scheme: the protocol scheme for the API endpoint. Normally + "https", but for testing or local servers, + "http" can be specified. + http_options: a dictionary of http_options for transcoding, to override + the defaults from operatons.proto. Each method has an entry + with the corresponding http rules as value. + + """ + # Run the base constructor + # TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc. + # TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the + # credentials object + super().__init__( + host=host, + credentials=credentials, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + ) + self._session = AuthorizedSession( + self._credentials, default_host=self.DEFAULT_HOST + ) + if client_cert_source_for_mtls: + self._session.configure_mtls_channel(client_cert_source_for_mtls) + self._prep_wrapped_messages(client_info) + self._http_options = http_options or {} + + def _list_operations( + self, + request: operations_pb2.ListOperationsRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> operations_pb2.ListOperationsResponse: + r"""Call the list operations method over HTTP. + + Args: + request (~.operations_pb2.ListOperationsRequest): + The request object. The request message for + [Operations.ListOperations][google.api_core.operations_v1.Operations.ListOperations]. + + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + ~.operations_pb2.ListOperationsResponse: + The response message for + [Operations.ListOperations][google.api_core.operations_v1.Operations.ListOperations]. + + """ + + http_options = [ + {"method": "get", "uri": "/v1/{name=operations}"}, + ] + if "google.longrunning.Operations.ListOperations" in self._http_options: + http_options = self._http_options[ + "google.longrunning.Operations.ListOperations" + ] + + request_kwargs = json_format.MessageToDict( + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + transcoded_request = path_template.transcode(http_options, **request_kwargs) + + uri = transcoded_request["uri"] + method = transcoded_request["method"] + + # Jsonify the query params + query_params_request = operations_pb2.ListOperationsRequest() + json_format.ParseDict(transcoded_request["query_params"], query_params_request) + query_params = json_format.MessageToDict( + query_params_request, + including_default_value_fields=False, + preserving_proto_field_name=False, + use_integers_for_enums=False, + ) + + # Send the request + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(self._session, method)( + "https://{host}{uri}".format(host=self._host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params), + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + # Return the response + api_response = operations_pb2.ListOperationsResponse() + json_format.Parse(response.content, api_response, ignore_unknown_fields=False) + return api_response + + def _get_operation( + self, + request: operations_pb2.GetOperationRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> operations_pb2.Operation: + r"""Call the get operation method over HTTP. + + Args: + request (~.operations_pb2.GetOperationRequest): + The request object. The request message for + [Operations.GetOperation][google.api_core.operations_v1.Operations.GetOperation]. + + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + ~.operations_pb2.Operation: + This resource represents a long- + unning operation that is the result of a + network API call. + + """ + + http_options = [ + {"method": "get", "uri": "/v1/{name=operations/**}"}, + ] + if "google.longrunning.Operations.GetOperation" in self._http_options: + http_options = self._http_options[ + "google.longrunning.Operations.GetOperation" + ] + + request_kwargs = json_format.MessageToDict( + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + transcoded_request = path_template.transcode(http_options, **request_kwargs) + + uri = transcoded_request["uri"] + method = transcoded_request["method"] + + # Jsonify the query params + query_params_request = operations_pb2.GetOperationRequest() + json_format.ParseDict(transcoded_request["query_params"], query_params_request) + query_params = json_format.MessageToDict( + query_params_request, + including_default_value_fields=False, + preserving_proto_field_name=False, + use_integers_for_enums=False, + ) + + # Send the request + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(self._session, method)( + "https://{host}{uri}".format(host=self._host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params), + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + # Return the response + api_response = operations_pb2.Operation() + json_format.Parse(response.content, api_response, ignore_unknown_fields=False) + return api_response + + def _delete_operation( + self, + request: operations_pb2.DeleteOperationRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> empty_pb2.Empty: + r"""Call the delete operation method over HTTP. + + Args: + request (~.operations_pb2.DeleteOperationRequest): + The request object. The request message for + [Operations.DeleteOperation][google.api_core.operations_v1.Operations.DeleteOperation]. + + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + """ + + http_options = [ + {"method": "delete", "uri": "/v1/{name=operations/**}"}, + ] + if "google.longrunning.Operations.DeleteOperation" in self._http_options: + http_options = self._http_options[ + "google.longrunning.Operations.DeleteOperation" + ] + + request_kwargs = json_format.MessageToDict( + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + transcoded_request = path_template.transcode(http_options, **request_kwargs) + + uri = transcoded_request["uri"] + method = transcoded_request["method"] + + # Jsonify the query params + query_params_request = operations_pb2.DeleteOperationRequest() + json_format.ParseDict(transcoded_request["query_params"], query_params_request) + query_params = json_format.MessageToDict( + query_params_request, + including_default_value_fields=False, + preserving_proto_field_name=False, + use_integers_for_enums=False, + ) + + # Send the request + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(self._session, method)( + "https://{host}{uri}".format(host=self._host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params), + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + return empty_pb2.Empty() + + def _cancel_operation( + self, + request: operations_pb2.CancelOperationRequest, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Optional[float] = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> empty_pb2.Empty: + r"""Call the cancel operation method over HTTP. + + Args: + request (~.operations_pb2.CancelOperationRequest): + The request object. The request message for + [Operations.CancelOperation][google.api_core.operations_v1.Operations.CancelOperation]. + + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + """ + + http_options = [ + {"method": "post", "uri": "/v1/{name=operations/**}:cancel", "body": "*"}, + ] + if "google.longrunning.Operations.CancelOperation" in self._http_options: + http_options = self._http_options[ + "google.longrunning.Operations.CancelOperation" + ] + + request_kwargs = json_format.MessageToDict( + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + transcoded_request = path_template.transcode(http_options, **request_kwargs) + + # Jsonify the request body + body_request = operations_pb2.CancelOperationRequest() + json_format.ParseDict(transcoded_request["body"], body_request) + body = json_format.MessageToDict( + body_request, + including_default_value_fields=False, + preserving_proto_field_name=False, + use_integers_for_enums=False, + ) + uri = transcoded_request["uri"] + method = transcoded_request["method"] + + # Jsonify the query params + query_params_request = operations_pb2.CancelOperationRequest() + json_format.ParseDict(transcoded_request["query_params"], query_params_request) + query_params = json_format.MessageToDict( + query_params_request, + including_default_value_fields=False, + preserving_proto_field_name=False, + use_integers_for_enums=False, + ) + + # Send the request + headers = dict(metadata) + headers["Content-Type"] = "application/json" + response = getattr(self._session, method)( + "https://{host}{uri}".format(host=self._host, uri=uri), + timeout=timeout, + headers=headers, + params=rest_helpers.flatten_query_params(query_params), + data=body, + ) + + # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception + # subclass. + if response.status_code >= 400: + raise core_exceptions.from_http_response(response) + + return empty_pb2.Empty() + + @property + def list_operations( + self, + ) -> Callable[ + [operations_pb2.ListOperationsRequest], operations_pb2.ListOperationsResponse + ]: + return self._list_operations + + @property + def get_operation( + self, + ) -> Callable[[operations_pb2.GetOperationRequest], operations_pb2.Operation]: + return self._get_operation + + @property + def delete_operation( + self, + ) -> Callable[[operations_pb2.DeleteOperationRequest], empty_pb2.Empty]: + return self._delete_operation + + @property + def cancel_operation( + self, + ) -> Callable[[operations_pb2.CancelOperationRequest], empty_pb2.Empty]: + return self._cancel_operation + + +__all__ = ("OperationsRestTransport",) diff --git a/tests/unit/operations_v1/test_operations_rest_client.py b/tests/unit/operations_v1/test_operations_rest_client.py new file mode 100644 index 00000000..dddf6b71 --- /dev/null +++ b/tests/unit/operations_v1/test_operations_rest_client.py @@ -0,0 +1,944 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +import mock +import pytest + +try: + import grpc # noqa: F401 +except ImportError: + pytest.skip("No GRPC", allow_module_level=True) +from requests import Response # noqa I201 +from requests.sessions import Session + +from google.api_core import client_options +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core.operations_v1 import AbstractOperationsClient +from google.api_core.operations_v1 import pagers +from google.api_core.operations_v1 import transports +import google.auth +from google.auth import credentials as ga_credentials +from google.auth.exceptions import MutualTLSChannelError +from google.longrunning import operations_pb2 +from google.oauth2 import service_account +from google.protobuf import json_format # type: ignore +from google.rpc import status_pb2 # type: ignore + + +HTTP_OPTIONS = { + "google.longrunning.Operations.CancelOperation": [ + {"method": "post", "uri": "/v3/{name=operations/*}:cancel", "body": "*"}, + ], + "google.longrunning.Operations.DeleteOperation": [ + {"method": "delete", "uri": "/v3/{name=operations/*}"}, + ], + "google.longrunning.Operations.GetOperation": [ + {"method": "get", "uri": "/v3/{name=operations/*}"}, + ], + "google.longrunning.Operations.ListOperations": [ + {"method": "get", "uri": "/v3/{name=operations}"}, + ], +} + + +def client_cert_source_callback(): + return b"cert bytes", b"key bytes" + + +def _get_operations_client(http_options=HTTP_OPTIONS): + transport = transports.rest.OperationsRestTransport( + credentials=ga_credentials.AnonymousCredentials(), http_options=http_options + ) + + return AbstractOperationsClient(transport=transport) + + +# If default endpoint is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint(client): + return ( + "foo.googleapis.com" + if ("localhost" in client.DEFAULT_ENDPOINT) + else client.DEFAULT_ENDPOINT + ) + + +def test__get_default_mtls_endpoint(): + api_endpoint = "example.googleapis.com" + api_mtls_endpoint = "example.mtls.googleapis.com" + sandbox_endpoint = "example.sandbox.googleapis.com" + sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" + non_googleapi = "api.example.com" + + assert AbstractOperationsClient._get_default_mtls_endpoint(None) is None + assert ( + AbstractOperationsClient._get_default_mtls_endpoint(api_endpoint) + == api_mtls_endpoint + ) + assert ( + AbstractOperationsClient._get_default_mtls_endpoint(api_mtls_endpoint) + == api_mtls_endpoint + ) + assert ( + AbstractOperationsClient._get_default_mtls_endpoint(sandbox_endpoint) + == sandbox_mtls_endpoint + ) + assert ( + AbstractOperationsClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) + == sandbox_mtls_endpoint + ) + assert ( + AbstractOperationsClient._get_default_mtls_endpoint(non_googleapi) + == non_googleapi + ) + + +@pytest.mark.parametrize("client_class", [AbstractOperationsClient]) +def test_operations_client_from_service_account_info(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object( + service_account.Credentials, "from_service_account_info" + ) as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == "longrunning.googleapis.com:443" + + +@pytest.mark.parametrize( + "transport_class,transport_name", [(transports.OperationsRestTransport, "rest")] +) +def test_operations_client_service_account_always_use_jwt( + transport_class, transport_name +): + with mock.patch.object( + service_account.Credentials, "with_always_use_jwt_access", create=True + ) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object( + service_account.Credentials, "with_always_use_jwt_access", create=True + ) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize("client_class", [AbstractOperationsClient]) +def test_operations_client_from_service_account_file(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object( + service_account.Credentials, "from_service_account_file" + ) as factory: + factory.return_value = creds + client = client_class.from_service_account_file("dummy/file/path.json") + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + client = client_class.from_service_account_json("dummy/file/path.json") + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == "longrunning.googleapis.com:443" + + +def test_operations_client_get_transport_class(): + transport = AbstractOperationsClient.get_transport_class() + available_transports = [ + transports.OperationsRestTransport, + ] + assert transport in available_transports + + transport = AbstractOperationsClient.get_transport_class("rest") + assert transport == transports.OperationsRestTransport + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name", + [(AbstractOperationsClient, transports.OperationsRestTransport, "rest")], +) +@mock.patch.object( + AbstractOperationsClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(AbstractOperationsClient), +) +def test_operations_client_client_options( + client_class, transport_class, transport_name +): + # Check that if channel is provided we won't create a new one. + with mock.patch.object(AbstractOperationsClient, "get_transport_class") as gtc: + transport = transport_class(credentials=ga_credentials.AnonymousCredentials()) + client = client_class(transport=transport) + gtc.assert_not_called() + + # Check that if channel is provided via str we will create a new one. + with mock.patch.object(AbstractOperationsClient, "get_transport_class") as gtc: + client = client_class(transport=transport_name) + gtc.assert_called() + + # Check the case api_endpoint is provided. + options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_MTLS_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError): + client = client_class() + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + with pytest.raises(ValueError): + client = client_class() + + # Check the case quota_project_id is provided + options = client_options.ClientOptions(quota_project_id="octopus") + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id="octopus", + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name,use_client_cert_env", + [ + (AbstractOperationsClient, transports.OperationsRestTransport, "rest", "true"), + (AbstractOperationsClient, transports.OperationsRestTransport, "rest", "false"), + ], +) +@mock.patch.object( + AbstractOperationsClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(AbstractOperationsClient), +) +@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) +def test_operations_client_mtls_env_auto( + client_class, transport_class, transport_name, use_client_cert_env +): + # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default + # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. + + # Check the case client_cert_source is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): + options = client_options.ClientOptions( + client_cert_source=client_cert_source_callback + ) + + def fake_init(client_cert_source_for_mtls=None, **kwargs): + """Invoke client_cert source if provided.""" + + if client_cert_source_for_mtls: + client_cert_source_for_mtls() + return None + + with mock.patch.object(transport_class, "__init__") as patched: + patched.side_effect = fake_init + client = client_class(client_options=options) + + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client.DEFAULT_ENDPOINT + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT + + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case ADC client cert is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): + with mock.patch.object(transport_class, "__init__") as patched: + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=client_cert_source_callback, + ): + if use_client_cert_env == "false": + expected_host = client.DEFAULT_ENDPOINT + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback + + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): + with mock.patch.object(transport_class, "__init__") as patched: + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=False, + ): + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name", + [(AbstractOperationsClient, transports.OperationsRestTransport, "rest")], +) +def test_operations_client_client_options_scopes( + client_class, transport_class, transport_name +): + # Check the case scopes are provided. + options = client_options.ClientOptions(scopes=["1", "2"],) + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=["1", "2"], + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name", + [(AbstractOperationsClient, transports.OperationsRestTransport, "rest")], +) +def test_operations_client_client_options_credentials_file( + client_class, transport_class, transport_name +): + # Check the case credentials file is provided. + options = client_options.ClientOptions(credentials_file="credentials.json") + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +def test_list_operations_rest( + transport: str = "rest", request_type=operations_pb2.ListOperationsRequest +): + client = _get_operations_client() + + # Mock the http request call within the method and fake a response. + with mock.patch.object(Session, "request") as req: + # Designate an appropriate value for the returned response. + return_value = operations_pb2.ListOperationsResponse( + next_page_token="next_page_token_value", + ) + + # Wrap the value into a proper Response obj + response_value = Response() + response_value.status_code = 200 + json_return_value = json_format.MessageToJson(return_value) + response_value._content = json_return_value.encode("UTF-8") + req.return_value = response_value + response = client.list_operations( + name="operations", filter_="my_filter", page_size=10, page_token="abc" + ) + + actual_args = req.call_args + assert actual_args.args[0] == "GET" + assert ( + actual_args.args[1] + == "https://longrunning.googleapis.com:443/v3/operations" + ) + assert actual_args.kwargs["params"] == [ + ("filter", "my_filter"), + ("pageSize", 10), + ("pageToken", "abc"), + ] + + # Establish that the response is the type that we expect. + assert isinstance(response, pagers.ListOperationsPager) + assert response.next_page_token == "next_page_token_value" + + +def test_list_operations_rest_failure(): + client = _get_operations_client(http_options=None) + + with mock.patch.object(Session, "request") as req: + response_value = Response() + response_value.status_code = 400 + mock_request = mock.MagicMock() + mock_request.method = "GET" + mock_request.url = "https://longrunning.googleapis.com:443/v1/operations" + response_value.request = mock_request + req.return_value = response_value + with pytest.raises(core_exceptions.GoogleAPIError): + client.list_operations(name="operations") + + +def test_list_operations_rest_pager(): + client = AbstractOperationsClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the http request call within the method and fake a response. + with mock.patch.object(Session, "request") as req: + # TODO(kbandes): remove this mock unless there's a good reason for it. + # with mock.patch.object(path_template, 'transcode') as transcode: + # Set the response as a series of pages + response = ( + operations_pb2.ListOperationsResponse( + operations=[ + operations_pb2.Operation(), + operations_pb2.Operation(), + operations_pb2.Operation(), + ], + next_page_token="abc", + ), + operations_pb2.ListOperationsResponse( + operations=[], next_page_token="def", + ), + operations_pb2.ListOperationsResponse( + operations=[operations_pb2.Operation()], next_page_token="ghi", + ), + operations_pb2.ListOperationsResponse( + operations=[operations_pb2.Operation(), operations_pb2.Operation()], + ), + ) + # Two responses for two calls + response = response + response + + # Wrap the values into proper Response objs + response = tuple(json_format.MessageToJson(x) for x in response) + return_values = tuple(Response() for i in response) + for return_val, response_val in zip(return_values, response): + return_val._content = response_val.encode("UTF-8") + return_val.status_code = 200 + req.side_effect = return_values + + pager = client.list_operations(name="operations") + + results = list(pager) + assert len(results) == 6 + assert all(isinstance(i, operations_pb2.Operation) for i in results) + + pages = list(client.list_operations(name="operations").pages) + for page_, token in zip(pages, ["abc", "def", "ghi", ""]): + assert page_.next_page_token == token + + +def test_get_operation_rest( + transport: str = "rest", request_type=operations_pb2.GetOperationRequest +): + client = _get_operations_client() + + # Mock the http request call within the method and fake a response. + with mock.patch.object(Session, "request") as req: + # Designate an appropriate value for the returned response. + return_value = operations_pb2.Operation( + name="operations/sample1", done=True, error=status_pb2.Status(code=411), + ) + + # Wrap the value into a proper Response obj + response_value = Response() + response_value.status_code = 200 + json_return_value = json_format.MessageToJson(return_value) + response_value._content = json_return_value.encode("UTF-8") + req.return_value = response_value + response = client.get_operation("operations/sample1") + + actual_args = req.call_args + assert actual_args.args[0] == "GET" + assert ( + actual_args.args[1] + == "https://longrunning.googleapis.com:443/v3/operations/sample1" + ) + + # Establish that the response is the type that we expect. + assert isinstance(response, operations_pb2.Operation) + assert response.name == "operations/sample1" + assert response.done is True + + +def test_get_operation_rest_failure(): + client = _get_operations_client(http_options=None) + + with mock.patch.object(Session, "request") as req: + response_value = Response() + response_value.status_code = 400 + mock_request = mock.MagicMock() + mock_request.method = "GET" + mock_request.url = ( + "https://longrunning.googleapis.com:443/v1/operations/sample1" + ) + response_value.request = mock_request + req.return_value = response_value + with pytest.raises(core_exceptions.GoogleAPIError): + client.get_operation("operations/sample1") + + +def test_delete_operation_rest( + transport: str = "rest", request_type=operations_pb2.DeleteOperationRequest +): + client = _get_operations_client() + + # Mock the http request call within the method and fake a response. + with mock.patch.object(Session, "request") as req: + # Wrap the value into a proper Response obj + response_value = Response() + response_value.status_code = 200 + json_return_value = "" + response_value._content = json_return_value.encode("UTF-8") + req.return_value = response_value + client.delete_operation(name="operations/sample1") + assert req.call_count == 1 + actual_args = req.call_args + assert actual_args.args[0] == "DELETE" + assert ( + actual_args.args[1] + == "https://longrunning.googleapis.com:443/v3/operations/sample1" + ) + + +def test_delete_operation_rest_failure(): + client = _get_operations_client(http_options=None) + + with mock.patch.object(Session, "request") as req: + response_value = Response() + response_value.status_code = 400 + mock_request = mock.MagicMock() + mock_request.method = "DELETE" + mock_request.url = ( + "https://longrunning.googleapis.com:443/v1/operations/sample1" + ) + response_value.request = mock_request + req.return_value = response_value + with pytest.raises(core_exceptions.GoogleAPIError): + client.delete_operation(name="operations/sample1") + + +def test_cancel_operation_rest(transport: str = "rest"): + client = _get_operations_client() + + # Mock the http request call within the method and fake a response. + with mock.patch.object(Session, "request") as req: + # Wrap the value into a proper Response obj + response_value = Response() + response_value.status_code = 200 + json_return_value = "" + response_value._content = json_return_value.encode("UTF-8") + req.return_value = response_value + client.cancel_operation(name="operations/sample1") + assert req.call_count == 1 + actual_args = req.call_args + assert actual_args.args[0] == "POST" + assert ( + actual_args.args[1] + == "https://longrunning.googleapis.com:443/v3/operations/sample1:cancel" + ) + + +def test_cancel_operation_rest_failure(): + client = _get_operations_client(http_options=None) + + with mock.patch.object(Session, "request") as req: + response_value = Response() + response_value.status_code = 400 + mock_request = mock.MagicMock() + mock_request.method = "POST" + mock_request.url = ( + "https://longrunning.googleapis.com:443/v1/operations/sample1:cancel" + ) + response_value.request = mock_request + req.return_value = response_value + with pytest.raises(core_exceptions.GoogleAPIError): + client.cancel_operation(name="operations/sample1") + + +def test_credentials_transport_error(): + # It is an error to provide credentials and a transport instance. + transport = transports.OperationsRestTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + AbstractOperationsClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # It is an error to provide a credentials file and a transport instance. + transport = transports.OperationsRestTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + AbstractOperationsClient( + client_options={"credentials_file": "credentials.json"}, + transport=transport, + ) + + # It is an error to provide scopes and a transport instance. + transport = transports.OperationsRestTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + AbstractOperationsClient( + client_options={"scopes": ["1", "2"]}, transport=transport, + ) + + +def test_transport_instance(): + # A client may be instantiated with a custom transport instance. + transport = transports.OperationsRestTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + client = AbstractOperationsClient(transport=transport) + assert client.transport is transport + + +@pytest.mark.parametrize("transport_class", [transports.OperationsRestTransport]) +def test_transport_adc(transport_class): + # Test default credentials are used if not provided. + with mock.patch.object(google.auth, "default") as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class() + adc.assert_called_once() + + +def test_operations_base_transport_error(): + # Passing both a credentials object and credentials_file should raise an error + with pytest.raises(core_exceptions.DuplicateCredentialArgs): + transports.OperationsTransport( + credentials=ga_credentials.AnonymousCredentials(), + credentials_file="credentials.json", + ) + + +def test_operations_base_transport(): + # Instantiate the base transport. + with mock.patch( + "google.api_core.operations_v1.transports.OperationsTransport.__init__" + ) as Transport: + Transport.return_value = None + transport = transports.OperationsTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Every method on the transport should just blindly + # raise NotImplementedError. + methods = ( + "list_operations", + "get_operation", + "delete_operation", + "cancel_operation", + ) + for method in methods: + with pytest.raises(NotImplementedError): + getattr(transport, method)(request=object()) + + with pytest.raises(NotImplementedError): + transport.close() + + +def test_operations_base_transport_with_credentials_file(): + # Instantiate the base transport with a credentials file + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch( + "google.api_core.operations_v1.transports.OperationsTransport._prep_wrapped_messages" + ) as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transports.OperationsTransport( + credentials_file="credentials.json", quota_project_id="octopus", + ) + load_creds.assert_called_once_with( + "credentials.json", + scopes=None, + default_scopes=(), + quota_project_id="octopus", + ) + + +def test_operations_base_transport_with_adc(): + # Test the default credentials are used if credentials and credentials_file are None. + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch( + "google.api_core.operations_v1.transports.OperationsTransport._prep_wrapped_messages" + ) as Transport: + Transport.return_value = None + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transports.OperationsTransport() + adc.assert_called_once() + + +def test_operations_auth_adc(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + AbstractOperationsClient() + adc.assert_called_once_with( + scopes=None, default_scopes=(), quota_project_id=None, + ) + + +def test_operations_http_transport_client_cert_source_for_mtls(): + cred = ga_credentials.AnonymousCredentials() + with mock.patch( + "google.auth.transport.requests.AuthorizedSession.configure_mtls_channel" + ) as mock_configure_mtls_channel: + transports.OperationsRestTransport( + credentials=cred, client_cert_source_for_mtls=client_cert_source_callback + ) + mock_configure_mtls_channel.assert_called_once_with(client_cert_source_callback) + + +def test_operations_host_no_port(): + client = AbstractOperationsClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions( + api_endpoint="longrunning.googleapis.com" + ), + ) + assert client.transport._host == "longrunning.googleapis.com:443" + + +def test_operations_host_with_port(): + client = AbstractOperationsClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions( + api_endpoint="longrunning.googleapis.com:8000" + ), + ) + assert client.transport._host == "longrunning.googleapis.com:8000" + + +def test_common_billing_account_path(): + billing_account = "squid" + expected = "billingAccounts/{billing_account}".format( + billing_account=billing_account, + ) + actual = AbstractOperationsClient.common_billing_account_path(billing_account) + assert expected == actual + + +def test_parse_common_billing_account_path(): + expected = { + "billing_account": "clam", + } + path = AbstractOperationsClient.common_billing_account_path(**expected) + + # Check that the path construction is reversible. + actual = AbstractOperationsClient.parse_common_billing_account_path(path) + assert expected == actual + + +def test_common_folder_path(): + folder = "whelk" + expected = "folders/{folder}".format(folder=folder,) + actual = AbstractOperationsClient.common_folder_path(folder) + assert expected == actual + + +def test_parse_common_folder_path(): + expected = { + "folder": "octopus", + } + path = AbstractOperationsClient.common_folder_path(**expected) + + # Check that the path construction is reversible. + actual = AbstractOperationsClient.parse_common_folder_path(path) + assert expected == actual + + +def test_common_organization_path(): + organization = "oyster" + expected = "organizations/{organization}".format(organization=organization,) + actual = AbstractOperationsClient.common_organization_path(organization) + assert expected == actual + + +def test_parse_common_organization_path(): + expected = { + "organization": "nudibranch", + } + path = AbstractOperationsClient.common_organization_path(**expected) + + # Check that the path construction is reversible. + actual = AbstractOperationsClient.parse_common_organization_path(path) + assert expected == actual + + +def test_common_project_path(): + project = "cuttlefish" + expected = "projects/{project}".format(project=project,) + actual = AbstractOperationsClient.common_project_path(project) + assert expected == actual + + +def test_parse_common_project_path(): + expected = { + "project": "mussel", + } + path = AbstractOperationsClient.common_project_path(**expected) + + # Check that the path construction is reversible. + actual = AbstractOperationsClient.parse_common_project_path(path) + assert expected == actual + + +def test_common_location_path(): + project = "winkle" + location = "nautilus" + expected = "projects/{project}/locations/{location}".format( + project=project, location=location, + ) + actual = AbstractOperationsClient.common_location_path(project, location) + assert expected == actual + + +def test_parse_common_location_path(): + expected = { + "project": "scallop", + "location": "abalone", + } + path = AbstractOperationsClient.common_location_path(**expected) + + # Check that the path construction is reversible. + actual = AbstractOperationsClient.parse_common_location_path(path) + assert expected == actual + + +def test_client_withDEFAULT_CLIENT_INFO(): + client_info = gapic_v1.client_info.ClientInfo() + + with mock.patch.object( + transports.OperationsTransport, "_prep_wrapped_messages" + ) as prep: + AbstractOperationsClient( + credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + with mock.patch.object( + transports.OperationsTransport, "_prep_wrapped_messages" + ) as prep: + transport_class = AbstractOperationsClient.get_transport_class() + transport_class( + credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, + ) + prep.assert_called_once_with(client_info) From cb091f8669124be83df78424afed442b37767acc Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 7 Dec 2021 15:18:37 -0800 Subject: [PATCH 043/112] chore: release 2.3.0 (#314) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 12 ++++++++++++ google/api_core/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70ce6afc..04d47daa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.3.0](https://www.github.com/googleapis/python-api-core/compare/v2.2.2...v2.3.0) (2021-11-25) + + +### Features + +* add operations rest client to support long-running operations. ([#311](https://www.github.com/googleapis/python-api-core/issues/311)) ([ce1adf3](https://www.github.com/googleapis/python-api-core/commit/ce1adf395982ede157c0f25a920946bb52789873)) + + +### Bug Fixes + +* handle bare 'grpc.Call' in 'from_grpc_error' ([#298](https://www.github.com/googleapis/python-api-core/issues/298)) ([060b339](https://www.github.com/googleapis/python-api-core/commit/060b339e3af296dd1772bfc1b4a0d2b4264cae1f)) + ### [2.2.2](https://www.github.com/googleapis/python-api-core/compare/v2.2.1...v2.2.2) (2021-11-02) diff --git a/google/api_core/version.py b/google/api_core/version.py index bcc59c44..999199f5 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.2.2" +__version__ = "2.3.0" From 479d6a7474f79947115d560f34fb971295c6af8b Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 8 Dec 2021 18:25:04 -0500 Subject: [PATCH 044/112] ci: run lint / mypy / unit tests / coverage as GH actions (#287) Make GH Action checks required --- .github/sync-repo-settings.yaml | 25 ++++++ .github/workflows/unittest_lint.yml | 119 ++++++++++++++++++++++++++++ noxfile.py | 2 +- 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/unittest_lint.yml diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index e621885d..67fcd59f 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -1,3 +1,28 @@ +# https://github.com/googleapis/repo-automation-bots/tree/main/packages/sync-repo-settings +# Rules for main branch protection +branchProtectionRules: +# Identifies the protection rule pattern. Name of the branch to be protected. +# Defaults to `main` +- pattern: main + requiresCodeOwnerReviews: true + requiresStrictStatusChecks: true + requiredStatusCheckContexts: + - 'cla/google' + # No Kokoro: the following are Github actions + - 'lint-mypy' + - 'unit-3.6' + - 'unit-3.7' + - 'unit-3.8' + - 'unit-3.9' + - 'unit-3.10' + - 'unit_grpc_gcp-3.6' + - 'unit_grpc_gcp-3.7' + - 'unit_grpc_gcp-3.8' + - 'unit_grpc_gcp-3.9' + - 'unit_grpc_gcp-3.10' + - 'unit_wo_grpc-3.6' + - 'unit_wo_grpc-3.10' + - 'cover' permissionRules: - team: actools-python permission: admin diff --git a/.github/workflows/unittest_lint.yml b/.github/workflows/unittest_lint.yml new file mode 100644 index 00000000..302be970 --- /dev/null +++ b/.github/workflows/unittest_lint.yml @@ -0,0 +1,119 @@ +name: "Lint / Unit tests / Cover / Mypy" + +on: + pull_request: + branches: + - main + + +jobs: + + run-lint-mypy: + name: lint-mypy + runs-on: ubuntu-latest + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + + - name: Run lint + run: | + nox -s lint + + - name: Run lint_setup_py + run: | + nox -s lint_setup_py + + - name: Run mypy + run: | + nox -s mypy + + run-unittests: + name: unit${{ matrix.option }}-${{ matrix.python }} + runs-on: ubuntu-latest + strategy: + matrix: + option: ["", "_grpc_gcp", "_wo_grpc"] + python: + - "3.6" + - "3.7" + - "3.8" + - "3.9" + - "3.10" + exclude: + - option: "_wo_grpc" + python: 3.7 + - option: "_wo_grpc" + python: 3.8 + - option: "_wo_grpc" + python: 3.9 + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + + - name: Run unit tests + env: + COVERAGE_FILE: .coverage${{ matrix.option }}-${{matrix.python }} + run: | + nox -s unit${{ matrix.option }}-${{ matrix.python }} + + - name: Upload coverage results + uses: actions/upload-artifact@v2 + with: + name: coverage-artifacts + path: .coverage${{ matrix.option }}-${{ matrix.python }} + + report-coverage: + name: cover + runs-on: ubuntu-latest + needs: + - run-unittests + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Install coverage + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install coverage + + - name: Download coverage results + uses: actions/download-artifact@v2 + with: + name: coverage-artifacts + path: .coverage-results/ + + - name: Report coverage results + run: | + coverage combine .coverage-results/.coverage* + coverage report --show-missing --fail-under=100 diff --git a/noxfile.py b/noxfile.py index db37c561..003a276d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -132,7 +132,7 @@ def unit(session): default(session) -@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) +@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) def unit_grpc_gcp(session): """Run the unit test suite with grpcio-gcp installed.""" constraints_path = str( From cc46aa68ec184871330d16a6c767f57a4f0eb633 Mon Sep 17 00:00:00 2001 From: Aza Tulepbergenov Date: Thu, 9 Dec 2021 10:32:49 -0800 Subject: [PATCH 045/112] feat: add support for 'error_info' (#315) * feat: Adds support for error_info. * chore: fixes pytype. Co-authored-by: Tres Seaver --- google/api_core/exceptions.py | 71 +++++++++++++++++++++++++++++++---- tests/unit/test_exceptions.py | 62 +++++++++++++++++++++++++----- 2 files changed, 116 insertions(+), 17 deletions(-) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 6b1b6f7e..24b65ee0 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -104,6 +104,8 @@ class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): details (Sequence[Any]): An optional list of objects defined in google.rpc.error_details. response (Union[requests.Request, grpc.Call]): The response or gRPC call metadata. + error_info (Union[error_details_pb2.ErrorInfo, None]): An optional object containing error info + (google.rpc.error_details.ErrorInfo). """ code: Union[int, None] = None @@ -122,13 +124,14 @@ class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): This may be ``None`` if the exception does not match up to a gRPC error. """ - def __init__(self, message, errors=(), details=(), response=None): + def __init__(self, message, errors=(), details=(), response=None, error_info=None): super(GoogleAPICallError, self).__init__(message) self.message = message """str: The exception message.""" self._errors = errors self._details = details self._response = response + self._error_info = error_info def __str__(self): if self.details: @@ -136,6 +139,42 @@ def __str__(self): else: return "{} {}".format(self.code, self.message) + @property + def reason(self): + """The reason of the error. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 + + Returns: + Union[str, None]: An optional string containing reason of the error. + """ + return self._error_info.reason if self._error_info else None + + @property + def domain(self): + """The logical grouping to which the "reason" belongs. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 + + Returns: + Union[str, None]: An optional string containing a logical grouping to which the "reason" belongs. + """ + return self._error_info.domain if self._error_info else None + + @property + def metadata(self): + """Additional structured details about this error. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 + + Returns: + Union[Dict[str, str], None]: An optional object containing structured details about the error. + """ + return self._error_info.metadata if self._error_info else None + @property def errors(self): """Detailed error information. @@ -433,13 +472,26 @@ def from_http_response(response): errors = payload.get("error", {}).get("errors", ()) # In JSON, details are already formatted in developer-friendly way. details = payload.get("error", {}).get("details", ()) + error_info = list( + filter( + lambda detail: detail.get("@type", "") + == "type.googleapis.com/google.rpc.ErrorInfo", + details, + ) + ) + error_info = error_info[0] if error_info else None message = "{method} {url}: {error}".format( - method=response.request.method, url=response.request.url, error=error_message + method=response.request.method, url=response.request.url, error=error_message, ) exception = from_http_status( - response.status_code, message, errors=errors, details=details, response=response + response.status_code, + message, + errors=errors, + details=details, + response=response, + error_info=error_info, ) return exception @@ -490,10 +542,10 @@ def _parse_grpc_error_details(rpc_exc): try: status = rpc_status.from_call(rpc_exc) except NotImplementedError: # workaround - return [] + return [], None if not status: - return [] + return [], None possible_errors = [ error_details_pb2.BadRequest, @@ -507,6 +559,7 @@ def _parse_grpc_error_details(rpc_exc): error_details_pb2.Help, error_details_pb2.LocalizedMessage, ] + error_info = None error_details = [] for detail in status.details: matched_detail_cls = list( @@ -519,7 +572,9 @@ def _parse_grpc_error_details(rpc_exc): info = matched_detail_cls[0]() detail.Unpack(info) error_details.append(info) - return error_details + if isinstance(info, error_details_pb2.ErrorInfo): + error_info = info + return error_details, error_info def from_grpc_error(rpc_exc): @@ -535,12 +590,14 @@ def from_grpc_error(rpc_exc): # NOTE(lidiz) All gRPC error shares the parent class grpc.RpcError. # However, check for grpc.RpcError breaks backward compatibility. if isinstance(rpc_exc, grpc.Call) or _is_informative_grpc_error(rpc_exc): + details, err_info = _parse_grpc_error_details(rpc_exc) return from_grpc_status( rpc_exc.code(), rpc_exc.details(), errors=(rpc_exc,), - details=_parse_grpc_error_details(rpc_exc), + details=details, response=rpc_exc, + error_info=err_info, ) else: return GoogleAPICallError(str(rpc_exc), errors=(rpc_exc,), response=rpc_exc) diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 622f58ab..4169ad44 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -275,31 +275,56 @@ def create_bad_request_details(): return status_detail +def create_error_info_details(): + info = error_details_pb2.ErrorInfo( + reason="SERVICE_DISABLED", + domain="googleapis.com", + metadata={ + "consumer": "projects/455411330361", + "service": "translate.googleapis.com", + }, + ) + status_detail = any_pb2.Any() + status_detail.Pack(info) + return status_detail + + def test_error_details_from_rest_response(): bad_request_detail = create_bad_request_details() + error_info_detail = create_error_info_details() status = status_pb2.Status() status.code = 3 status.message = ( "3 INVALID_ARGUMENT: One of content, or gcs_content_uri must be set." ) status.details.append(bad_request_detail) + status.details.append(error_info_detail) # See JSON schema in https://cloud.google.com/apis/design/errors#http_mapping http_response = make_response( - json.dumps({"error": json.loads(json_format.MessageToJson(status))}).encode( - "utf-8" - ) + json.dumps( + {"error": json.loads(json_format.MessageToJson(status, sort_keys=True))} + ).encode("utf-8") ) exception = exceptions.from_http_response(http_response) - want_error_details = [json.loads(json_format.MessageToJson(bad_request_detail))] + want_error_details = [ + json.loads(json_format.MessageToJson(bad_request_detail)), + json.loads(json_format.MessageToJson(error_info_detail)), + ] assert want_error_details == exception.details + # 404 POST comes from make_response. assert str(exception) == ( "404 POST https://example.com/: 3 INVALID_ARGUMENT:" " One of content, or gcs_content_uri must be set." " [{'@type': 'type.googleapis.com/google.rpc.BadRequest'," - " 'fieldViolations': [{'field': 'document.content'," - " 'description': 'Must have some text content to annotate.'}]}]" + " 'fieldViolations': [{'description': 'Must have some text content to annotate.'," + " 'field': 'document.content'}]}," + " {'@type': 'type.googleapis.com/google.rpc.ErrorInfo'," + " 'domain': 'googleapis.com'," + " 'metadata': {'consumer': 'projects/455411330361'," + " 'service': 'translate.googleapis.com'}," + " 'reason': 'SERVICE_DISABLED'}]" ) @@ -311,6 +336,11 @@ def test_error_details_from_v1_rest_response(): ) exception = exceptions.from_http_response(response) assert exception.details == [] + assert ( + exception.reason is None + and exception.domain is None + and exception.metadata is None + ) @pytest.mark.skipif(grpc is None, reason="gRPC not importable") @@ -320,8 +350,10 @@ def test_error_details_from_grpc_response(): status.message = ( "3 INVALID_ARGUMENT: One of content, or gcs_content_uri must be set." ) - status_detail = create_bad_request_details() - status.details.append(status_detail) + status_br_detail = create_bad_request_details() + status_ei_detail = create_error_info_details() + status.details.append(status_br_detail) + status.details.append(status_ei_detail) # Actualy error doesn't matter as long as its grpc.Call, # because from_call is mocked. @@ -331,8 +363,13 @@ def test_error_details_from_grpc_response(): exception = exceptions.from_grpc_error(error) bad_request_detail = error_details_pb2.BadRequest() - status_detail.Unpack(bad_request_detail) - assert exception.details == [bad_request_detail] + error_info_detail = error_details_pb2.ErrorInfo() + status_br_detail.Unpack(bad_request_detail) + status_ei_detail.Unpack(error_info_detail) + assert exception.details == [bad_request_detail, error_info_detail] + assert exception.reason == error_info_detail.reason + assert exception.domain == error_info_detail.domain + assert exception.metadata == error_info_detail.metadata @pytest.mark.skipif(grpc is None, reason="gRPC not importable") @@ -351,3 +388,8 @@ def test_error_details_from_grpc_response_unknown_error(): m.return_value = status exception = exceptions.from_grpc_error(error) assert exception.details == [status_detail] + assert ( + exception.reason is None + and exception.domain is None + and exception.metadata is None + ) From 34ebdcc251d4f3d7d496e8e0b78847645a06650b Mon Sep 17 00:00:00 2001 From: Chris Wilson <46912004+sushicw@users.noreply.github.com> Date: Wed, 15 Dec 2021 02:12:56 -0800 Subject: [PATCH 046/112] fix: exclude function target from retry deadline exceeded exception message (#318) * Exclude function target from retry deadline exceeded exception message * apply similar patch in retry_async.py Co-authored-by: Anthonios Partheniou --- google/api_core/retry.py | 4 ++-- google/api_core/retry_async.py | 4 ++-- tests/asyncio/test_retry_async.py | 4 ++++ tests/unit/test_retry.py | 4 ++++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/google/api_core/retry.py b/google/api_core/retry.py index bd3a4a65..ce496937 100644 --- a/google/api_core/retry.py +++ b/google/api_core/retry.py @@ -203,8 +203,8 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None): if deadline_datetime is not None: if deadline_datetime <= now: raise exceptions.RetryError( - "Deadline of {:.1f}s exceeded while calling {}".format( - deadline, target + "Deadline of {:.1f}s exceeded while calling target function".format( + deadline ), last_exc, ) from last_exc diff --git a/google/api_core/retry_async.py b/google/api_core/retry_async.py index 2dfa2f6e..68a25597 100644 --- a/google/api_core/retry_async.py +++ b/google/api_core/retry_async.py @@ -132,8 +132,8 @@ async def retry_target(target, predicate, sleep_generator, deadline, on_error=No # Chains the raising RetryError with the root cause error, # which helps observability and debugability. raise exceptions.RetryError( - "Deadline of {:.1f}s exceeded while calling {}".format( - deadline, target + "Deadline of {:.1f}s exceeded while calling target function".format( + deadline ), last_exc, ) from last_exc diff --git a/tests/asyncio/test_retry_async.py b/tests/asyncio/test_retry_async.py index 9e51044b..873caaf1 100644 --- a/tests/asyncio/test_retry_async.py +++ b/tests/asyncio/test_retry_async.py @@ -120,6 +120,10 @@ async def test_retry_target_deadline_exceeded(utcnow, sleep): assert exc_info.match("last exception: meep") assert target.call_count == 2 + # Ensure the exception message does not include the target fn: + # it may be a partial with user data embedded + assert str(target) not in exc_info.exconly() + @pytest.mark.asyncio async def test_retry_target_bad_sleep_generator(): diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 199ca559..74c5d77c 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -152,6 +152,10 @@ def test_retry_target_deadline_exceeded(utcnow, sleep): assert exc_info.match("last exception: meep") assert target.call_count == 2 + # Ensure the exception message does not include the target fn: + # it may be a partial with user data embedded + assert str(target) not in exc_info.exconly() + def test_retry_target_bad_sleep_generator(): with pytest.raises(ValueError, match="Sleep generator"): From 6538e0322a82831c82646cf29524bd913cf624b6 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 6 Jan 2022 16:06:17 +0000 Subject: [PATCH 047/112] chore: use python-samples-reviewers (#328) --- .github/.OwlBot.lock.yaml | 2 +- .github/CODEOWNERS | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 7519fa3a..f33299dd 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:0e18b9475fbeb12d9ad4302283171edebb6baf2dfca1bd215ee3b34ed79d95d7 + digest: sha256:899d5d7cc340fa8ef9d8ae1a8cfba362c6898584f779e156f25ee828ba824610 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ee818917..1b023b72 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,5 +8,5 @@ # @googleapis/yoshi-python @googleapis/actools-python are the default owners for changes in this repo * @googleapis/yoshi-python @googleapis/actools-python -# @googleapis/python-samples-owners @googleapis/actools-python are the default owners for samples changes -/samples/ @googleapis/python-samples-owners @googleapis/actools-python +# @googleapis/python-samples-reviewers @googleapis/actools-python are the default owners for samples changes +/samples/ @googleapis/python-samples-reviewers @googleapis/actools-python From 69a99d8dfd9492d3d449de99845d3f134ce1f40c Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 7 Jan 2022 13:48:52 -0500 Subject: [PATCH 048/112] chore: update release_level in repo-metadata.json (#326) * chore: update .repo-metadata.json * revert * remove api_shortname --- .repo-metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.repo-metadata.json b/.repo-metadata.json index e16c9d27..0f0abd93 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -2,7 +2,7 @@ "name": "google-api-core", "name_pretty": "Google API client core library", "client_documentation": "https://googleapis.dev/python/google-api-core/latest", - "release_level": "ga", + "release_level": "stable", "language": "python", "library_type": "CORE", "repo": "googleapis/python-api-core", From f9f26969842b456ea372bed941d712b7a9ab7239 Mon Sep 17 00:00:00 2001 From: Aza Tulepbergenov Date: Mon, 10 Jan 2022 09:31:35 -0800 Subject: [PATCH 049/112] feat: iterator for processing JSON responses in REST streaming. (#317) * feat: files for REST server streaming. --- google/api_core/rest_streaming.py | 114 ++++++++++++++++ tests/unit/test_rest_streaming.py | 211 ++++++++++++++++++++++++++++++ 2 files changed, 325 insertions(+) create mode 100644 google/api_core/rest_streaming.py create mode 100644 tests/unit/test_rest_streaming.py diff --git a/google/api_core/rest_streaming.py b/google/api_core/rest_streaming.py new file mode 100644 index 00000000..69f5b41b --- /dev/null +++ b/google/api_core/rest_streaming.py @@ -0,0 +1,114 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for server-side streaming in REST.""" + +from collections import deque +import string +from typing import Deque + +import requests + + +class ResponseIterator: + """Iterator over REST API responses. + + Args: + response (requests.Response): An API response object. + response_message_cls (Callable[proto.Message]): A proto + class expected to be returned from an API. + """ + + def __init__(self, response: requests.Response, response_message_cls): + self._response = response + self._response_message_cls = response_message_cls + # Inner iterator over HTTP response's content. + self._response_itr = self._response.iter_content(decode_unicode=True) + # Contains a list of JSON responses ready to be sent to user. + self._ready_objs: Deque[str] = deque() + # Current JSON response being built. + self._obj = "" + # Keeps track of the nesting level within a JSON object. + self._level = 0 + # Keeps track whether HTTP response is currently sending values + # inside of a string value. + self._in_string = False + # Whether an escape symbol "\" was encountered. + self._escape_next = False + + def cancel(self): + """Cancel existing streaming operation. + """ + self._response.close() + + def _process_chunk(self, chunk: str): + if self._level == 0: + if chunk[0] != "[": + raise ValueError( + "Can only parse array of JSON objects, instead got %s" % chunk + ) + for char in chunk: + if char == "{": + if self._level == 1: + # Level 1 corresponds to the outermost JSON object + # (i.e. the one we care about). + self._obj = "" + if not self._in_string: + self._level += 1 + self._obj += char + elif char == "}": + self._obj += char + if not self._in_string: + self._level -= 1 + if not self._in_string and self._level == 1: + self._ready_objs.append(self._obj) + elif char == '"': + # Helps to deal with an escaped quotes inside of a string. + if not self._escape_next: + self._in_string = not self._in_string + self._obj += char + elif char in string.whitespace: + if self._in_string: + self._obj += char + elif char == "[": + if self._level == 0: + self._level += 1 + else: + self._obj += char + elif char == "]": + if self._level == 1: + self._level -= 1 + else: + self._obj += char + else: + self._obj += char + self._escape_next = not self._escape_next if char == "\\" else False + + def __next__(self): + while not self._ready_objs: + try: + chunk = next(self._response_itr) + self._process_chunk(chunk) + except StopIteration as e: + if self._level > 0: + raise ValueError("Unfinished stream: %s" % self._obj) + raise e + return self._grab() + + def _grab(self): + # Add extra quotes to make json.loads happy. + return self._response_message_cls.from_json(self._ready_objs.popleft()) + + def __iter__(self): + return self diff --git a/tests/unit/test_rest_streaming.py b/tests/unit/test_rest_streaming.py new file mode 100644 index 00000000..4be59580 --- /dev/null +++ b/tests/unit/test_rest_streaming.py @@ -0,0 +1,211 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import random +import time +from typing import List +from unittest.mock import patch + +import proto +import pytest +import requests + +from google.api_core import rest_streaming +from google.protobuf import duration_pb2 +from google.protobuf import timestamp_pb2 + + +SEED = int(time.time()) +logging.info(f"Starting rest streaming tests with random seed: {SEED}") +random.seed(SEED) + + +class Genre(proto.Enum): + GENRE_UNSPECIFIED = 0 + CLASSICAL = 1 + JAZZ = 2 + ROCK = 3 + + +class Composer(proto.Message): + given_name = proto.Field(proto.STRING, number=1) + family_name = proto.Field(proto.STRING, number=2) + relateds = proto.RepeatedField(proto.STRING, number=3) + indices = proto.MapField(proto.STRING, proto.STRING, number=4) + + +class Song(proto.Message): + composer = proto.Field(Composer, number=1) + title = proto.Field(proto.STRING, number=2) + lyrics = proto.Field(proto.STRING, number=3) + year = proto.Field(proto.INT32, number=4) + genre = proto.Field(Genre, number=5) + is_five_mins_longer = proto.Field(proto.BOOL, number=6) + score = proto.Field(proto.DOUBLE, number=7) + likes = proto.Field(proto.INT64, number=8) + duration = proto.Field(duration_pb2.Duration, number=9) + date_added = proto.Field(timestamp_pb2.Timestamp, number=10) + + +class EchoResponse(proto.Message): + content = proto.Field(proto.STRING, number=1) + + +class ResponseMock(requests.Response): + class _ResponseItr: + def __init__(self, _response_bytes: bytes, random_split=False): + self._responses_bytes = _response_bytes + self._i = 0 + self._random_split = random_split + + def __next__(self): + if self._i == len(self._responses_bytes): + raise StopIteration + if self._random_split: + n = random.randint(1, len(self._responses_bytes[self._i :])) + else: + n = 1 + x = self._responses_bytes[self._i : self._i + n] + self._i += n + return x.decode("utf-8") + + def __init__( + self, responses: List[proto.Message], response_cls, random_split=False, + ): + super().__init__() + self._responses = responses + self._random_split = random_split + self._response_message_cls = response_cls + + def _parse_responses(self, responses: List[proto.Message]) -> bytes: + # json.dumps returns a string surrounded with quotes that need to be stripped + # in order to be an actual JSON. + json_responses = [ + self._response_message_cls.to_json(r).strip('"') for r in responses + ] + logging.info(f"Sending JSON stream: {json_responses}") + ret_val = "[{}]".format(",".join(json_responses)) + return bytes(ret_val, "utf-8") + + def close(self): + raise NotImplementedError() + + def iter_content(self, *args, **kwargs): + return self._ResponseItr( + self._parse_responses(self._responses), random_split=self._random_split, + ) + + +@pytest.mark.parametrize("random_split", [False]) +def test_next_simple(random_split): + responses = [EchoResponse(content="hello world"), EchoResponse(content="yes")] + resp = ResponseMock( + responses=responses, random_split=random_split, response_cls=EchoResponse + ) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + assert list(itr) == responses + + +@pytest.mark.parametrize("random_split", [True, False]) +def test_next_nested(random_split): + responses = [ + Song(title="some song", composer=Composer(given_name="some name")), + Song(title="another song", date_added=datetime.datetime(2021, 12, 17)), + ] + resp = ResponseMock( + responses=responses, random_split=random_split, response_cls=Song + ) + itr = rest_streaming.ResponseIterator(resp, Song) + assert list(itr) == responses + + +@pytest.mark.parametrize("random_split", [True, False]) +def test_next_stress(random_split): + n = 50 + responses = [ + Song(title="title_%d" % i, composer=Composer(given_name="name_%d" % i)) + for i in range(n) + ] + resp = ResponseMock( + responses=responses, random_split=random_split, response_cls=Song + ) + itr = rest_streaming.ResponseIterator(resp, Song) + assert list(itr) == responses + + +@pytest.mark.parametrize("random_split", [True, False]) +def test_next_escaped_characters_in_string(random_split): + composer_with_relateds = Composer() + relateds = ["Artist A", "Artist B"] + composer_with_relateds.relateds = relateds + + responses = [ + Song(title='ti"tle\nfoo\tbar{}', composer=Composer(given_name="name\n\n\n")), + Song( + title='{"this is weird": "totally"}', composer=Composer(given_name="\\{}\\") + ), + Song(title='\\{"key": ["value",]}\\', composer=composer_with_relateds), + ] + resp = ResponseMock( + responses=responses, random_split=random_split, response_cls=Song + ) + itr = rest_streaming.ResponseIterator(resp, Song) + assert list(itr) == responses + + +def test_next_not_array(): + with patch.object( + ResponseMock, "iter_content", return_value=iter('{"hello": 0}') + ) as mock_method: + + resp = ResponseMock(responses=[], response_cls=EchoResponse) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + with pytest.raises(ValueError): + next(itr) + mock_method.assert_called_once() + + +def test_cancel(): + with patch.object(ResponseMock, "close", return_value=None) as mock_method: + resp = ResponseMock(responses=[], response_cls=EchoResponse) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + itr.cancel() + mock_method.assert_called_once() + + +def test_check_buffer(): + with patch.object( + ResponseMock, + "_parse_responses", + return_value=bytes('[{"content": "hello"}, {', "utf-8"), + ): + resp = ResponseMock(responses=[], response_cls=EchoResponse) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + with pytest.raises(ValueError): + next(itr) + next(itr) + + +def test_next_html(): + with patch.object( + ResponseMock, "iter_content", return_value=iter("") + ) as mock_method: + + resp = ResponseMock(responses=[], response_cls=EchoResponse) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + with pytest.raises(ValueError): + next(itr) + mock_method.assert_called_once() From beb3e6317faf24ed87680ff73517826d910ca415 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 11 Jan 2022 18:35:46 -0500 Subject: [PATCH 050/112] chore: release 2.4.0 (#316) * chore: release 2.4.0 * chore: delete #318 from log (already released) * Update changelog to include published versions * insert newline in changelog * insert newline in changelog Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Co-authored-by: Anthonios Partheniou --- CHANGELOG.md | 21 +++++++++++++++++++++ google/api_core/version.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04d47daa..e6177e0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,27 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.4.0](https://www.github.com/googleapis/python-api-core/compare/v2.3.2...v2.4.0) (2022-01-11) + + +### Features + +* add support for 'error_info' ([#315](https://www.github.com/googleapis/python-api-core/issues/315)) ([cc46aa6](https://www.github.com/googleapis/python-api-core/commit/cc46aa68ec184871330d16a6c767f57a4f0eb633)) +* iterator for processing JSON responses in REST streaming. ([#317](https://www.github.com/googleapis/python-api-core/issues/317)) ([f9f2696](https://www.github.com/googleapis/python-api-core/commit/f9f26969842b456ea372bed941d712b7a9ab7239)) + +## [2.3.2](https://www.github.com/googleapis/python-api-core/compare/v2.3.1...v2.3.2) (2021-12-16) + + +### Bug Fixes + +* address broken wheels in version 2.3.1 + +## [2.3.1](https://www.github.com/googleapis/python-api-core/compare/v2.3.0...v2.3.1) (2021-12-15) + + +### Bug Fixes +* exclude function target from retry deadline exceeded exception message ([#318](https://www.github.com/googleapis/python-api-core/issues/318)) ([34ebdcc](https://www.github.com/googleapis/python-api-core/commit/34ebdcc251d4f3d7d496e8e0b78847645a06650b)) + ## [2.3.0](https://www.github.com/googleapis/python-api-core/compare/v2.2.2...v2.3.0) (2021-11-25) diff --git a/google/api_core/version.py b/google/api_core/version.py index 999199f5..fe11624d 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.3.0" +__version__ = "2.4.0" From adbe63844ed07c1b75e2e6dd97ff865a25750201 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 13 Jan 2022 15:54:21 +0000 Subject: [PATCH 051/112] build: switch to release-please for tagging (#331) --- .github/.OwlBot.lock.yaml | 2 +- .github/release-please.yml | 1 + .github/release-trigger.yml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .github/release-trigger.yml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index f33299dd..ff5126c1 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:899d5d7cc340fa8ef9d8ae1a8cfba362c6898584f779e156f25ee828ba824610 + digest: sha256:dfa9b663b32de8b5b327e32c1da665a80de48876558dd58091d8160c60ad7355 diff --git a/.github/release-please.yml b/.github/release-please.yml index 4507ad05..466597e5 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -1 +1,2 @@ releaseType: python +handleGHRelease: true diff --git a/.github/release-trigger.yml b/.github/release-trigger.yml new file mode 100644 index 00000000..d4ca9418 --- /dev/null +++ b/.github/release-trigger.yml @@ -0,0 +1 @@ +enabled: true From f267111823545a6c67ef5f10b85cd8c2fab8a612 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 14 Jan 2022 10:41:21 -0500 Subject: [PATCH 052/112] docs: fix typo in library name (#332) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d94f3e88..13d8d3a6 100644 --- a/README.rst +++ b/README.rst @@ -25,4 +25,4 @@ Unsupported Python Versions Python == 2.7, Python == 3.5. The last version of this library compatible with Python 2.7 and 3.5 is -`google-api_core==1.31.1`. +`google-api-core==1.31.1`. From 8169f2bfaaf59042a8b77d2bad9c5a4cc7ed3826 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 12:07:49 -0500 Subject: [PATCH 053/112] chore(python): update release.sh to use keystore (#333) Source-Link: https://github.com/googleapis/synthtool/commit/69fda12e2994f0b595a397e8bb6e3e9f380524eb Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/release.sh | 2 +- .kokoro/release/common.cfg | 12 +++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ff5126c1..eecb84c2 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:dfa9b663b32de8b5b327e32c1da665a80de48876558dd58091d8160c60ad7355 + digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 0728ce11..a00f93ec 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -26,7 +26,7 @@ python3 -m pip install --upgrade twine wheel setuptools export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token") +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1") cd github/python-api-core python3 setup.py sdist bdist_wheel twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index 586e7649..de4f6f89 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -23,8 +23,18 @@ env_vars: { value: "github/python-api-core/.kokoro/release.sh" } +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} + # Tokens needed to report release status back to GitHub env_vars: { key: "SECRET_MANAGER_KEYS" - value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } From 74d5ce8a20aa6e270de75fa5ac37c0bb26df7b45 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 17 Jan 2022 12:04:45 -0500 Subject: [PATCH 054/112] chore: move docs check from kokoro to GH actions (#334) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: move docs check from kokoro to GH actions * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix typo Co-authored-by: Owl Bot --- .github/sync-repo-settings.yaml | 1 + .github/workflows/docs.yml | 25 ++++++++++ .github/workflows/lint.yml | 30 ++++++++++++ .../{unittest_lint.yml => unittest.yml} | 47 +------------------ 4 files changed, 57 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/lint.yml rename .github/workflows/{unittest_lint.yml => unittest.yml} (76%) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 67fcd59f..db2ddd08 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -23,6 +23,7 @@ branchProtectionRules: - 'unit_wo_grpc-3.6' - 'unit_wo_grpc-3.10' - 'cover' + - 'docs' permissionRules: - team: actools-python permission: admin diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..bd90e5bf --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,25 @@ +name: "Docs" + +on: + pull_request: + branches: + - main +jobs: + run-docs: + name: docs + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.7" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run docs + run: | + nox -s docs docfx + diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..c6d94b00 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,30 @@ +name: "Lint" + +on: + pull_request: + branches: + - main +jobs: + run-lint-mypy: + name: lint-mypy + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.7" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run lint + run: | + nox -s lint + - name: Run lint_setup_py + run: | + nox -s lint_setup_py + - name: Run mypy + run: | + nox -s mypy diff --git a/.github/workflows/unittest_lint.yml b/.github/workflows/unittest.yml similarity index 76% rename from .github/workflows/unittest_lint.yml rename to .github/workflows/unittest.yml index 302be970..9ab0c395 100644 --- a/.github/workflows/unittest_lint.yml +++ b/.github/workflows/unittest.yml @@ -1,44 +1,11 @@ -name: "Lint / Unit tests / Cover / Mypy" +name: "Unit tests" on: pull_request: branches: - main - jobs: - - run-lint-mypy: - name: lint-mypy - runs-on: ubuntu-latest - - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: "3.10" - - - name: Install nox - run: | - python -m pip install --upgrade setuptools pip wheel - python -m pip install nox - - - name: Run lint - run: | - nox -s lint - - - name: Run lint_setup_py - run: | - nox -s lint_setup_py - - - name: Run mypy - run: | - nox -s mypy - run-unittests: name: unit${{ matrix.option }}-${{ matrix.python }} runs-on: ubuntu-latest @@ -58,28 +25,22 @@ jobs: python: 3.8 - option: "_wo_grpc" python: 3.9 - steps: - - name: Checkout uses: actions/checkout@v2 - - name: Setup Python uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} - - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel python -m pip install nox - - name: Run unit tests env: COVERAGE_FILE: .coverage${{ matrix.option }}-${{matrix.python }} run: | nox -s unit${{ matrix.option }}-${{ matrix.python }} - - name: Upload coverage results uses: actions/upload-artifact@v2 with: @@ -91,28 +52,22 @@ jobs: runs-on: ubuntu-latest needs: - run-unittests - steps: - - name: Checkout uses: actions/checkout@v2 - - name: Setup Python uses: actions/setup-python@v2 with: python-version: "3.10" - - name: Install coverage run: | python -m pip install --upgrade setuptools pip wheel python -m pip install coverage - - name: Download coverage results uses: actions/download-artifact@v2 with: name: coverage-artifacts path: .coverage-results/ - - name: Report coverage results run: | coverage combine .coverage-results/.coverage* From 5e5ad37b8161109d65b0fab43636f7424e570fa3 Mon Sep 17 00:00:00 2001 From: arithmetic1728 <58957152+arithmetic1728@users.noreply.github.com> Date: Wed, 19 Jan 2022 14:55:31 -0800 Subject: [PATCH 055/112] feat: add api_key to client options (#248) * feat: add api_key to client options * update --- google/api_core/client_options.py | 9 +++++++- tests/unit/test_client_options.py | 35 +++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/google/api_core/client_options.py b/google/api_core/client_options.py index be5523df..4a2a84a4 100644 --- a/google/api_core/client_options.py +++ b/google/api_core/client_options.py @@ -66,11 +66,14 @@ class ClientOptions(object): quota_project_id (Optional[str]): A project name that a client's quota belongs to. credentials_file (Optional[str]): A path to a file storing credentials. + ``credentials_file` and ``api_key`` are mutually exclusive. scopes (Optional[Sequence[str]]): OAuth access token override scopes. + api_key (Optional[str]): Google API key. ``credentials_file`` and + ``api_key`` are mutually exclusive. Raises: ValueError: If both ``client_cert_source`` and ``client_encrypted_cert_source`` - are provided. + are provided, or both ``credentials_file`` and ``api_key`` are provided. """ def __init__( @@ -81,17 +84,21 @@ def __init__( quota_project_id=None, credentials_file=None, scopes=None, + api_key=None, ): if client_cert_source and client_encrypted_cert_source: raise ValueError( "client_cert_source and client_encrypted_cert_source are mutually exclusive" ) + if api_key and credentials_file: + raise ValueError("api_key and credentials_file are mutually exclusive") self.api_endpoint = api_endpoint self.client_cert_source = client_cert_source self.client_encrypted_cert_source = client_encrypted_cert_source self.quota_project_id = quota_project_id self.credentials_file = credentials_file self.scopes = scopes + self.api_key = api_key def __repr__(self): return "ClientOptions: " + repr(self.__dict__) diff --git a/tests/unit/test_client_options.py b/tests/unit/test_client_options.py index 38b9ad0a..fbff5457 100644 --- a/tests/unit/test_client_options.py +++ b/tests/unit/test_client_options.py @@ -72,6 +72,36 @@ def test_constructor_with_both_cert_sources(): ) +def test_constructor_with_api_key(): + + options = client_options.ClientOptions( + api_endpoint="foo.googleapis.com", + client_cert_source=get_client_cert, + quota_project_id="quote-proj", + api_key="api-key", + scopes=[ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + ], + ) + + assert options.api_endpoint == "foo.googleapis.com" + assert options.client_cert_source() == (b"cert", b"key") + assert options.quota_project_id == "quote-proj" + assert options.api_key == "api-key" + assert options.scopes == [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + ] + + +def test_constructor_with_both_api_key_and_credentials_file(): + with pytest.raises(ValueError): + client_options.ClientOptions( + api_key="api-key", credentials_file="path/to/credentials.json", + ) + + def test_from_dict(): options = client_options.from_dict( { @@ -94,6 +124,7 @@ def test_from_dict(): "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/cloud-platform.read-only", ] + assert options.api_key is None def test_from_dict_bad_argument(): @@ -112,6 +143,6 @@ def test_repr(): assert ( repr(options) - == "ClientOptions: {'api_endpoint': 'foo.googleapis.com', 'client_cert_source': None, 'client_encrypted_cert_source': None}" - or "ClientOptions: {'client_encrypted_cert_source': None, 'client_cert_source': None, 'api_endpoint': 'foo.googleapis.com'}" + == "ClientOptions: {'api_endpoint': 'foo.googleapis.com', 'client_cert_source': None, 'client_encrypted_cert_source': None, 'api_key': None}" + or "ClientOptions: {'client_encrypted_cert_source': None, 'client_cert_source': None, 'api_endpoint': 'foo.googleapis.com', 'api_key': None}" ) From 95e41ff2715cc092d47be1e6244ae835c71e0acd Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Sat, 22 Jan 2022 06:04:24 -0500 Subject: [PATCH 056/112] ci(python): run lint / unit tests / docs as GH actions (#336) * ci(python): run lint / unit tests / docs as GH actions Source-Link: https://github.com/googleapis/synthtool/commit/57be0cdb0b94e1669cee0ca38d790de1dfdbcd44 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ed1f9983d5a935a89fe8085e8bb97d94e41015252c5b6c9771257cf8624367e6 * revert changes to unittest gh action * move mypy check into separate workflow * update .sync-repo-settings to reflect changes to gh checks * use python 3.7 for lint check Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 15 ++++++++++++++- .github/sync-repo-settings.yaml | 4 +++- .github/workflows/docs.yml | 27 ++++++++++++++++++++------- .github/workflows/lint.yml | 9 ++------- .github/workflows/mypy.yml | 22 ++++++++++++++++++++++ owlbot.py | 4 ++++ 6 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/mypy.yml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index eecb84c2..8cb43804 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,16 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 + digest: sha256:ed1f9983d5a935a89fe8085e8bb97d94e41015252c5b6c9771257cf8624367e6 diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index db2ddd08..49c96ed4 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -9,7 +9,8 @@ branchProtectionRules: requiredStatusCheckContexts: - 'cla/google' # No Kokoro: the following are Github actions - - 'lint-mypy' + - 'lint' + - 'mypy' - 'unit-3.6' - 'unit-3.7' - 'unit-3.8' @@ -24,6 +25,7 @@ branchProtectionRules: - 'unit_wo_grpc-3.10' - 'cover' - 'docs' + - 'docfx' permissionRules: - team: actools-python permission: admin diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bd90e5bf..f7b8344c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,12 +1,10 @@ -name: "Docs" - on: pull_request: branches: - main +name: docs jobs: - run-docs: - name: docs + docs: runs-on: ubuntu-latest steps: - name: Checkout @@ -14,12 +12,27 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.10" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel python -m pip install nox - name: Run docs run: | - nox -s docs docfx - + nox -s docs + docfx: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run docfx + run: | + nox -s docfx diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c6d94b00..79ef73c2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,12 +1,10 @@ -name: "Lint" - on: pull_request: branches: - main +name: lint jobs: - run-lint-mypy: - name: lint-mypy + lint: runs-on: ubuntu-latest steps: - name: Checkout @@ -25,6 +23,3 @@ jobs: - name: Run lint_setup_py run: | nox -s lint_setup_py - - name: Run mypy - run: | - nox -s mypy diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml new file mode 100644 index 00000000..f3212da7 --- /dev/null +++ b/.github/workflows/mypy.yml @@ -0,0 +1,22 @@ +on: + pull_request: + branches: + - main +name: mypy +jobs: + mypy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.7" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run mypy + run: | + nox -s mypy diff --git a/owlbot.py b/owlbot.py index 451f7c48..0b7f7d21 100644 --- a/owlbot.py +++ b/owlbot.py @@ -28,6 +28,7 @@ ".flake8", # flake8-import-order, layout ".coveragerc", # layout "CONTRIBUTING.rst", # no systests + ".github/workflows/unittest.yml", # exclude unittest gh action ] templated_files = common.py_library(microgenerator=True, cov_level=100) s.move(templated_files, excludes=excludes) @@ -44,4 +45,7 @@ """, ) +s.replace(".github/workflows/lint.yml", "python-version: \"3.10\"", "python-version: \"3.7\"") + + s.shell.run(["nox", "-s", "blacken"], hide_output=False) From c782f294b50b078f01959627fb82aa4c5efec333 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Wed, 2 Feb 2022 13:19:37 -0700 Subject: [PATCH 057/112] fix(deps): remove setuptools from dependencies (#339) * fix(deps): remove setuptools from dependencies Fixes #338 * chore: remove setuptools from constraints --- setup.py | 1 - testing/constraints-3.6.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/setup.py b/setup.py index ddc56004..e400a52f 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,6 @@ "protobuf >= 3.12.0", "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", - "setuptools >= 40.3.0", ] extras = { "grpc": ["grpcio >= 1.33.2, < 2.0dev", "grpcio-status >= 1.33.2, < 2.0dev"], diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 0c2a07b6..bb1eb1e3 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -9,7 +9,6 @@ googleapis-common-protos==1.52.0 protobuf==3.12.0 google-auth==1.25.0 requests==2.18.0 -setuptools==40.3.0 packaging==14.3 grpcio==1.33.2 grpcio-gcp==0.2.2 From 4422ccedd46fb1fd1e99844381681d8aa314c90c Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 2 Feb 2022 15:31:14 -0700 Subject: [PATCH 058/112] chore(main): release 2.5.0 (#335) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 17 +++++++++++++++++ google/api_core/version.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6177e0e..6cbede4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.5.0](https://github.com/googleapis/python-api-core/compare/v2.4.0...v2.5.0) (2022-02-02) + + +### Features + +* add api_key to client options ([#248](https://github.com/googleapis/python-api-core/issues/248)) ([5e5ad37](https://github.com/googleapis/python-api-core/commit/5e5ad37b8161109d65b0fab43636f7424e570fa3)) + + +### Bug Fixes + +* **deps:** remove setuptools from dependencies ([#339](https://github.com/googleapis/python-api-core/issues/339)) ([c782f29](https://github.com/googleapis/python-api-core/commit/c782f294b50b078f01959627fb82aa4c5efec333)) + + +### Documentation + +* fix typo in library name ([#332](https://github.com/googleapis/python-api-core/issues/332)) ([f267111](https://github.com/googleapis/python-api-core/commit/f267111823545a6c67ef5f10b85cd8c2fab8a612)) + ## [2.4.0](https://www.github.com/googleapis/python-api-core/compare/v2.3.2...v2.4.0) (2022-01-11) diff --git a/google/api_core/version.py b/google/api_core/version.py index fe11624d..5836d805 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.4.0" +__version__ = "2.5.0" From b558e3d7e2fec3abb5f9ced0e6be232639233458 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 1 Mar 2022 12:26:13 +0000 Subject: [PATCH 059/112] chore(deps): update actions/setup-python action to v3 (#346) Source-Link: https://github.com/googleapis/synthtool/commit/571ee2c3b26182429eddcf115122ee545d7d3787 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:660abdf857d3ab9aabcd967c163c70e657fcc5653595c709263af5f3fa23ef67 --- .github/.OwlBot.lock.yaml | 2 +- .github/workflows/docs.yml | 4 ++-- .github/workflows/lint.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 8cb43804..d9a55fa4 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ed1f9983d5a935a89fe8085e8bb97d94e41015252c5b6c9771257cf8624367e6 + digest: sha256:660abdf857d3ab9aabcd967c163c70e657fcc5653595c709263af5f3fa23ef67 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f7b8344c..cca4e98b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.10" - name: Install nox @@ -26,7 +26,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.10" - name: Install nox diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 79ef73c2..f5331399 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.7" - name: Install nox From 6f614911fa4554838359c54096212ac93692f546 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 2 Mar 2022 17:08:08 +0100 Subject: [PATCH 060/112] chore(deps): update all dependencies to v3 (#347) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies to v3 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- .github/workflows/mypy.yml | 4 ++-- .github/workflows/unittest.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index f3212da7..e67dcbc5 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,9 +8,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.7" - name: Install nox diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 9ab0c395..1a857a3d 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -27,9 +27,9 @@ jobs: python: 3.9 steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python }} - name: Install nox @@ -54,9 +54,9 @@ jobs: - run-unittests steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.10" - name: Install coverage From 021bb7d5bf0a1d8ac58dbf0c738fac309135ba7d Mon Sep 17 00:00:00 2001 From: Dov Shlachter Date: Wed, 2 Mar 2022 10:28:44 -0800 Subject: [PATCH 061/112] feat: initial support for Extended Operations (#344) Certain APIs with Long-Running Operations deviate from the semantics in https://google.aip.dev/151 and instead define custom operation messages, aka Extended Operations. This change adds a PollingFuture subclass designed to be used with Extended Operations. It is analogous and broadly similar to google.api_core.operation.Operation and subclasses google.api_core.future.polling.PollingFuture. The full description of Extended Operation semantics is beyond the scope of this change. --- google/api_core/extended_operation.py | 206 ++++++++++++++++++++++++++ noxfile.py | 46 +++--- tests/unit/test_extended_operation.py | 182 +++++++++++++++++++++++ 3 files changed, 417 insertions(+), 17 deletions(-) create mode 100644 google/api_core/extended_operation.py create mode 100644 tests/unit/test_extended_operation.py diff --git a/google/api_core/extended_operation.py b/google/api_core/extended_operation.py new file mode 100644 index 00000000..209f1213 --- /dev/null +++ b/google/api_core/extended_operation.py @@ -0,0 +1,206 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Futures for extended long-running operations returned from Google Cloud APIs. + +These futures can be used to synchronously wait for the result of a +long-running operations using :meth:`ExtendedOperation.result`: + +.. code-block:: python + + extended_operation = my_api_client.long_running_method() + + extended_operation.result() + +Or asynchronously using callbacks and :meth:`Operation.add_done_callback`: + +.. code-block:: python + + extended_operation = my_api_client.long_running_method() + + def my_callback(ex_op): + print(f"Operation {ex_op.name} completed") + + extended_operation.add_done_callback(my_callback) + +""" + +import threading + +from google.api_core import exceptions +from google.api_core.future import polling + + +class ExtendedOperation(polling.PollingFuture): + """An ExtendedOperation future for interacting with a Google API Long-Running Operation. + + Args: + extended_operation (proto.Message): The initial operation. + refresh (Callable[[], type(extended_operation)]): A callable that returns + the latest state of the operation. + cancel (Callable[[], None]): A callable that tries to cancel the operation. + retry: Optional(google.api_core.retry.Retry): The retry configuration used + when polling. This can be used to control how often :meth:`done` + is polled. Regardless of the retry's ``deadline``, it will be + overridden by the ``timeout`` argument to :meth:`result`. + + Note: Most long-running API methods use google.api_core.operation.Operation + This class is a wrapper for a subset of methods that use alternative + Long-Running Operation (LRO) semantics. + + Note: there is not a concrete type the extended operation must be. + It MUST have fields that correspond to the following, POSSIBLY WITH DIFFERENT NAMES: + * name: str + * status: Union[str, bool, enum.Enum] + * error_code: int + * error_message: str + """ + + def __init__( + self, extended_operation, refresh, cancel, retry=polling.DEFAULT_RETRY + ): + super().__init__(retry=retry) + self._extended_operation = extended_operation + self._refresh = refresh + self._cancel = cancel + # Note: the extended operation does not give a good way to indicate cancellation. + # We make do with manually tracking cancellation and checking for doneness. + self._cancelled = False + self._completion_lock = threading.Lock() + # Invoke in case the operation came back already complete. + self._handle_refreshed_operation() + + # Note: the following four properties MUST be overridden in a subclass + # if, and only if, the fields in the corresponding extended operation message + # have different names. + # + # E.g. we have an extended operation class that looks like + # + # class MyOperation(proto.Message): + # moniker = proto.Field(proto.STRING, number=1) + # status_msg = proto.Field(proto.STRING, number=2) + # optional http_error_code = proto.Field(proto.INT32, number=3) + # optional http_error_msg = proto.Field(proto.STRING, number=4) + # + # the ExtendedOperation subclass would provide property overrrides that map + # to these (poorly named) fields. + @property + def name(self): + return self._extended_operation.name + + @property + def status(self): + return self._extended_operation.status + + @property + def error_code(self): + return self._extended_operation.error_code + + @property + def error_message(self): + return self._extended_operation.error_message + + def done(self, retry=polling.DEFAULT_RETRY): + self._refresh_and_update(retry) + return self._extended_operation.done + + def cancel(self): + if self.done(): + return False + + self._cancel() + self._cancelled = True + return True + + def cancelled(self): + # TODO(dovs): there is not currently a good way to determine whether the + # operation has been cancelled. + # The best we can do is manually keep track of cancellation + # and check for doneness. + if not self._cancelled: + return False + + self._refresh_and_update() + return self._extended_operation.done + + def _refresh_and_update(self, retry=polling.DEFAULT_RETRY): + if not self._extended_operation.done: + self._extended_operation = self._refresh(retry=retry) + self._handle_refreshed_operation() + + def _handle_refreshed_operation(self): + with self._completion_lock: + if not self._extended_operation.done: + return + + if self.error_code and self.error_message: + exception = exceptions.from_http_status( + status_code=self.error_code, + message=self.error_message, + response=self._extended_operation, + ) + self.set_exception(exception) + elif self.error_code or self.error_message: + exception = exceptions.GoogleAPICallError( + f"Unexpected error {self.error_code}: {self.error_message}" + ) + self.set_exception(exception) + else: + # Extended operations have no payload. + self.set_result(None) + + @classmethod + def make(cls, refresh, cancel, extended_operation, **kwargs): + """ + Return an instantiated ExtendedOperation (or child) that wraps + * a refresh callable + * a cancel callable (can be a no-op) + * an initial result + + .. note:: + It is the caller's responsibility to set up refresh and cancel + with their correct request argument. + The reason for this is that the services that use Extended Operations + have rpcs that look something like the following: + + // service.proto + service MyLongService { + rpc StartLongTask(StartLongTaskRequest) returns (ExtendedOperation) { + option (google.cloud.operation_service) = "CustomOperationService"; + } + } + + service CustomOperationService { + rpc Get(GetOperationRequest) returns (ExtendedOperation) { + option (google.cloud.operation_polling_method) = true; + } + } + + Any info needed for the poll, e.g. a name, path params, etc. + is held in the request, which the initial client method is in a much + better position to make made because the caller made the initial request. + + TL;DR: the caller sets up closures for refresh and cancel that carry + the properly configured requests. + + Args: + refresh (Callable[Optional[Retry]][type(extended_operation)]): A callable that + returns the latest state of the operation. + cancel (Callable[][Any]): A callable that tries to cancel the operation + on a best effort basis. + extended_operation (Any): The initial response of the long running method. + See the docstring for ExtendedOperation.__init__ for requirements on + the type and fields of extended_operation + """ + return cls(extended_operation, refresh, cancel, **kwargs) diff --git a/noxfile.py b/noxfile.py index 003a276d..e02a1280 100644 --- a/noxfile.py +++ b/noxfile.py @@ -92,7 +92,7 @@ def default(session, install_grpc=True): ) # Install all test dependencies, then install this package in-place. - session.install("mock", "pytest", "pytest-cov") + session.install("dataclasses", "mock", "pytest", "pytest-cov", "pytest-xdist") if install_grpc: session.install("-e", ".[grpc]", "-c", constraints_path) else: @@ -102,28 +102,36 @@ def default(session, install_grpc=True): "python", "-m", "py.test", - "--quiet", - "--cov=google.api_core", - "--cov=tests.unit", - "--cov-append", - "--cov-config=.coveragerc", - "--cov-report=", - "--cov-fail-under=0", - os.path.join("tests", "unit"), + *( + # Helpful for running a single test or testfile. + session.posargs + or [ + "--quiet", + "--cov=google.api_core", + "--cov=tests.unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + # Running individual tests with parallelism enabled is usually not helpful. + "-n=auto", + os.path.join("tests", "unit"), + ] + ), ] - pytest_args.extend(session.posargs) # Inject AsyncIO content and proto-plus, if version >= 3.6. # proto-plus is needed for a field mask test in test_protobuf_helpers.py if _greater_or_equal_than_36(session.python): session.install("asyncmock", "pytest-asyncio", "proto-plus") - pytest_args.append("--cov=tests.asyncio") - pytest_args.append(os.path.join("tests", "asyncio")) - session.run(*pytest_args) - else: - # Run py.test against the unit tests. - session.run(*pytest_args) + # Having positional arguments means the user wants to run specific tests. + # Best not to add additional tests to that list. + if not session.posargs: + pytest_args.append("--cov=tests.asyncio") + pytest_args.append(os.path.join("tests", "asyncio")) + + session.run(*pytest_args) @nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) @@ -171,7 +179,11 @@ def mypy(session): """Run type-checking.""" session.install(".[grpc, grpcgcp]", "mypy") session.install( - "types-setuptools", "types-requests", "types-protobuf", "types-mock" + "types-setuptools", + "types-requests", + "types-protobuf", + "types-mock", + "types-dataclasses", ) session.run("mypy", "google", "tests") diff --git a/tests/unit/test_extended_operation.py b/tests/unit/test_extended_operation.py new file mode 100644 index 00000000..7fcebed8 --- /dev/null +++ b/tests/unit/test_extended_operation.py @@ -0,0 +1,182 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataclasses +import enum +import typing + +import mock +import pytest + +from google.api_core import exceptions +from google.api_core import extended_operation +from google.api_core import retry + +TEST_OPERATION_NAME = "test/extended_operation" + + +@dataclasses.dataclass(frozen=True) +class CustomOperation: + class StatusCode(enum.Enum): + UNKNOWN = 0 + DONE = 1 + PENDING = 2 + + name: str + status: StatusCode + error_code: typing.Optional[int] = None + error_message: typing.Optional[str] = None + + # Note: in generated clients, this property must be generated for each + # extended operation message type. + # The status may be an enum, a string, or a bool. If it's a string or enum, + # its text is compared to the string "DONE". + @property + def done(self): + return self.status.name == "DONE" + + +def make_extended_operation(responses=None): + client_operations_responses = responses or [ + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.PENDING + ) + ] + + refresh = mock.Mock(spec=["__call__"], side_effect=client_operations_responses) + refresh.responses = client_operations_responses + cancel = mock.Mock(spec=["__call__"]) + extended_operation_future = extended_operation.ExtendedOperation.make( + refresh, cancel, client_operations_responses[0], + ) + + return extended_operation_future, refresh, cancel + + +def test_constructor(): + ex_op, refresh, _ = make_extended_operation() + assert ex_op._extended_operation == refresh.responses[0] + assert ex_op.cancelled() is False + assert ex_op.done() is False + assert ex_op.name == TEST_OPERATION_NAME + assert ex_op.status == CustomOperation.StatusCode.PENDING + assert ex_op.error_code is None + assert ex_op.error_message is None + + +def test_done(): + responses = [ + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.PENDING + ), + # Second response indicates that the operation has finished. + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.DONE + ), + # Bumper to make sure we stop polling on DONE. + CustomOperation( + name=TEST_OPERATION_NAME, + status=CustomOperation.StatusCode.DONE, + error_message="Gone too far!", + ), + ] + ex_op, refresh, _ = make_extended_operation(responses) + + # Start out not done. + assert not ex_op.done() + assert refresh.call_count == 1 + + # Refresh brings us to the done state. + assert ex_op.done() + assert refresh.call_count == 2 + assert not ex_op.error_message + + # Make sure that subsequent checks are no-ops. + assert ex_op.done() + assert refresh.call_count == 2 + assert not ex_op.error_message + + +def test_cancellation(): + responses = [ + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.PENDING + ), + # Second response indicates that the operation was cancelled. + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.DONE + ), + ] + ex_op, _, cancel = make_extended_operation(responses) + + assert not ex_op.cancelled() + + assert ex_op.cancel() + assert ex_op.cancelled() + cancel.assert_called_once_with() + + # Cancelling twice should have no effect. + assert not ex_op.cancel() + cancel.assert_called_once_with() + + +def test_done_w_retry(): + # Not sure what's going on here with the coverage, so just ignore it. + test_retry = retry.Retry(predicate=lambda x: True) # pragma: NO COVER + + responses = [ + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.PENDING + ), + CustomOperation( + name=TEST_OPERATION_NAME, status=CustomOperation.StatusCode.DONE + ), + ] + + ex_op, refresh, _ = make_extended_operation(responses) + + ex_op.done(retry=test_retry) + + refresh.assert_called_once_with(retry=test_retry) + + +def test_error(): + responses = [ + CustomOperation( + name=TEST_OPERATION_NAME, + status=CustomOperation.StatusCode.DONE, + error_code=400, + error_message="Bad request", + ), + ] + + ex_op, _, _ = make_extended_operation(responses) + + # Defaults to CallError when grpc is not installed + with pytest.raises(exceptions.BadRequest): + ex_op.result() + + # Inconsistent result + responses = [ + CustomOperation( + name=TEST_OPERATION_NAME, + status=CustomOperation.StatusCode.DONE, + error_code=2112, + ), + ] + + ex_op, _, _ = make_extended_operation(responses) + + with pytest.raises(exceptions.GoogleAPICallError): + ex_op.result() From a226a7c4ef0bae9a7b2cd03e3ac9aff32eaaff0b Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 3 Mar 2022 00:28:36 +0000 Subject: [PATCH 062/112] chore(deps): update actions/checkout action to v3 (#349) Source-Link: https://github.com/googleapis/synthtool/commit/ca879097772aeec2cbb971c3cea8ecc81522b68a Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:6162c384d685c5fe22521d3f37f6fc732bf99a085f6d47b677dbcae97fc21392 --- .github/.OwlBot.lock.yaml | 2 +- .github/workflows/docs.yml | 4 ++-- .github/workflows/lint.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index d9a55fa4..480226ac 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:660abdf857d3ab9aabcd967c163c70e657fcc5653595c709263af5f3fa23ef67 + digest: sha256:6162c384d685c5fe22521d3f37f6fc732bf99a085f6d47b677dbcae97fc21392 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index cca4e98b..b46d7305 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f5331399..2fce29f0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: From 04a8b8ba833909602603d73519047a2effeb6a22 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 3 Mar 2022 11:44:12 -0800 Subject: [PATCH 063/112] chore(main): release 2.6.0 (#350) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cbede4c..d8e41ac8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.6.0](https://github.com/googleapis/python-api-core/compare/v2.5.0...v2.6.0) (2022-03-03) + + +### Features + +* initial support for Extended Operations ([#344](https://github.com/googleapis/python-api-core/issues/344)) ([021bb7d](https://github.com/googleapis/python-api-core/commit/021bb7d5bf0a1d8ac58dbf0c738fac309135ba7d)) + ## [2.5.0](https://github.com/googleapis/python-api-core/compare/v2.4.0...v2.5.0) (2022-02-02) diff --git a/google/api_core/version.py b/google/api_core/version.py index 5836d805..ae34a9fb 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.5.0" +__version__ = "2.6.0" From 7e21e9e34892472a34f9b44175fa761f0e3fd9ed Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sat, 5 Mar 2022 18:37:00 +0200 Subject: [PATCH 064/112] fix: Remove py2 tag from wheel (#343) Co-authored-by: Anthonios Partheniou --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 0be0b3ff..8857e9b0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[bdist_wheel] -universal = 1 - [pytype] python_version = 3.6 inputs = From 07747ca9b25601f19ade161d3730eef1da13e2ee Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 11:05:45 -0800 Subject: [PATCH 065/112] chore(main): release 2.6.1 (#352) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e41ac8..b70b67ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.6.1](https://github.com/googleapis/python-api-core/compare/v2.6.0...v2.6.1) (2022-03-05) + + +### Bug Fixes + +* Remove py2 tag from wheel ([#343](https://github.com/googleapis/python-api-core/issues/343)) ([7e21e9e](https://github.com/googleapis/python-api-core/commit/7e21e9e34892472a34f9b44175fa761f0e3fd9ed)) + ## [2.6.0](https://github.com/googleapis/python-api-core/compare/v2.5.0...v2.6.0) (2022-03-03) diff --git a/google/api_core/version.py b/google/api_core/version.py index ae34a9fb..410cd066 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.6.0" +__version__ = "2.6.1" From 9abc6f48f23c87b9771dca3c96b4f6af39620a50 Mon Sep 17 00:00:00 2001 From: Dov Shlachter Date: Mon, 7 Mar 2022 16:09:34 -0800 Subject: [PATCH 066/112] feat: expose extra fields in ExtendedOperation (#351) The operation wrapped by ExtendedOperation may define other fields or methods that the user may wish to use. --- google/api_core/extended_operation.py | 3 +++ tests/unit/test_extended_operation.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/google/api_core/extended_operation.py b/google/api_core/extended_operation.py index 209f1213..cabae107 100644 --- a/google/api_core/extended_operation.py +++ b/google/api_core/extended_operation.py @@ -111,6 +111,9 @@ def error_code(self): def error_message(self): return self._extended_operation.error_message + def __getattr__(self, name): + return getattr(self._extended_operation, name) + def done(self, retry=polling.DEFAULT_RETRY): self._refresh_and_update(retry) return self._extended_operation.done diff --git a/tests/unit/test_extended_operation.py b/tests/unit/test_extended_operation.py index 7fcebed8..9521fde8 100644 --- a/tests/unit/test_extended_operation.py +++ b/tests/unit/test_extended_operation.py @@ -37,6 +37,7 @@ class StatusCode(enum.Enum): status: StatusCode error_code: typing.Optional[int] = None error_message: typing.Optional[str] = None + armor_class: typing.Optional[int] = None # Note: in generated clients, this property must be generated for each # extended operation message type. @@ -180,3 +181,23 @@ def test_error(): with pytest.raises(exceptions.GoogleAPICallError): ex_op.result() + + +def test_pass_through(): + responses = [ + CustomOperation( + name=TEST_OPERATION_NAME, + status=CustomOperation.StatusCode.PENDING, + armor_class=10, + ), + CustomOperation( + name=TEST_OPERATION_NAME, + status=CustomOperation.StatusCode.DONE, + armor_class=20, + ), + ] + ex_op, _, _ = make_extended_operation(responses) + + assert ex_op.armor_class == 10 + ex_op.result() + assert ex_op.armor_class == 20 From ea41551806f76c6737747f9e4864525b03746682 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 8 Mar 2022 10:15:47 -0800 Subject: [PATCH 067/112] chore(main): release 2.7.0 (#353) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b70b67ed..d7964273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.7.0](https://github.com/googleapis/python-api-core/compare/v2.6.1...v2.7.0) (2022-03-08) + + +### Features + +* expose extra fields in ExtendedOperation ([#351](https://github.com/googleapis/python-api-core/issues/351)) ([9abc6f4](https://github.com/googleapis/python-api-core/commit/9abc6f48f23c87b9771dca3c96b4f6af39620a50)) + ### [2.6.1](https://github.com/googleapis/python-api-core/compare/v2.6.0...v2.6.1) (2022-03-05) diff --git a/google/api_core/version.py b/google/api_core/version.py index 410cd066..d962613e 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.6.1" +__version__ = "2.7.0" From 0680fb4d3e013fe2de27e0a2ae2cd9896479e596 Mon Sep 17 00:00:00 2001 From: Aza Tulepbergenov Date: Wed, 9 Mar 2022 13:35:53 -0800 Subject: [PATCH 068/112] fix: add more context to error message. (#340) Co-authored-by: Anthonios Partheniou --- google/api_core/path_template.py | 7 +++++-- tests/unit/test_path_template.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/google/api_core/path_template.py b/google/api_core/path_template.py index 41fbd4fe..b6284c85 100644 --- a/google/api_core/path_template.py +++ b/google/api_core/path_template.py @@ -270,7 +270,6 @@ def transcode(http_options, **request_kwargs): ] path_args = {field: get_field(request_kwargs, field) for field in path_fields} request["uri"] = expand(uri_template, **path_args) - # Remove fields used in uri path from request leftovers = copy.deepcopy(request_kwargs) for path_field in path_fields: @@ -297,4 +296,8 @@ def transcode(http_options, **request_kwargs): request["method"] = http_option["method"] return request - raise ValueError("Request obj does not match any template") + raise ValueError( + "Request {} does not match any URL path template in available HttpRule's {}".format( + request_kwargs, [opt["uri"] for opt in http_options] + ) + ) diff --git a/tests/unit/test_path_template.py b/tests/unit/test_path_template.py index 2c5216e0..c12b35fc 100644 --- a/tests/unit/test_path_template.py +++ b/tests/unit/test_path_template.py @@ -362,6 +362,7 @@ def test_transcode_with_additional_bindings( [[["get", "/v1/{name}", ""]], {"name": "first/last"}], [[["get", "/v1/{name=mr/*/*}", ""]], {"name": "first/last"}], [[["post", "/v1/{name}", "data"]], {"name": "first/last"}], + [[["post", "/v1/{first_name}", "data"]], {"last_name": "last"}], ], ) def test_transcode_fails(http_options, request_kwargs): @@ -385,5 +386,4 @@ def helper_test_transcode(http_options_list, expected_result_list): } if expected_result_list[2]: expected_result["body"] = expected_result_list[2] - return (http_options, expected_result) From c89f55d10df50c222d05ea046e07e05ef7d17725 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 9 Mar 2022 13:37:46 -0800 Subject: [PATCH 069/112] chore(main): release 2.7.1 (#354) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7964273..0610df01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.7.1](https://github.com/googleapis/python-api-core/compare/v2.7.0...v2.7.1) (2022-03-09) + + +### Bug Fixes + +* add more context to error message. ([#340](https://github.com/googleapis/python-api-core/issues/340)) ([0680fb4](https://github.com/googleapis/python-api-core/commit/0680fb4d3e013fe2de27e0a2ae2cd9896479e596)) + ## [2.7.0](https://github.com/googleapis/python-api-core/compare/v2.6.1...v2.7.0) (2022-03-08) diff --git a/google/api_core/version.py b/google/api_core/version.py index d962613e..a97caa56 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.7.0" +__version__ = "2.7.1" From 112049e79f5a5b0a989d85d438a1bd29485f46f7 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 15 Mar 2022 11:35:14 -0400 Subject: [PATCH 070/112] fix: allow grpc without grpcio-status (#355) --- google/api_core/exceptions.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 24b65ee0..38fe6e78 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -24,15 +24,23 @@ import http.client from typing import Dict from typing import Union +import warnings from google.rpc import error_details_pb2 try: import grpc - from grpc_status import rpc_status + + try: + from grpc_status import rpc_status + except ImportError: # pragma: NO COVER + warnings.warn( + "Please install grpcio-status to obtain helpful grpc error messages.", + ImportWarning, + ) + rpc_status = None except ImportError: # pragma: NO COVER grpc = None - rpc_status = None # Lookup tables for mapping exceptions from HTTP and gRPC transports. # Populated by _GoogleAPICallErrorMeta From 778d27d18ce17e7c0b6dbc1fb56ddf093e278a4e Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 30 Mar 2022 06:18:58 -0400 Subject: [PATCH 071/112] chore(python): use black==22.3.0 (#362) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(python): use black==22.3.0 Source-Link: https://github.com/googleapis/synthtool/commit/6fab84af09f2cf89a031fd8671d1def6b2931b11 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:7cffbc10910c3ab1b852c05114a08d374c195a81cdec1d4a67a1d129331d0bfe * ci: use black 22.3.0 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 2 +- docs/conf.py | 13 +++- google/api_core/datetime_helpers.py | 4 +- google/api_core/exceptions.py | 6 +- google/api_core/grpc_helpers.py | 4 +- .../abstract_operations_client.py | 68 +++++++++++++++---- .../api_core/operations_v1/transports/base.py | 6 +- google/api_core/path_template.py | 40 +++++------ google/api_core/rest_streaming.py | 3 +- noxfile.py | 7 +- tests/asyncio/gapic/test_method_async.py | 3 +- .../test_operations_rest_client.py | 48 +++++++++---- tests/unit/test_client_options.py | 3 +- tests/unit/test_extended_operation.py | 4 +- tests/unit/test_page_iterator.py | 3 +- tests/unit/test_rest_streaming.py | 8 ++- 16 files changed, 152 insertions(+), 70 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 480226ac..87dd0061 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:6162c384d685c5fe22521d3f37f6fc732bf99a085f6d47b677dbcae97fc21392 + digest: sha256:7cffbc10910c3ab1b852c05114a08d374c195a81cdec1d4a67a1d129331d0bfe diff --git a/docs/conf.py b/docs/conf.py index 09f0c2b6..9a80171b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -314,7 +314,13 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (root_doc, "google-api-core", "google-api-core Documentation", [author], 1,) + ( + root_doc, + "google-api-core", + "google-api-core Documentation", + [author], + 1, + ) ] # If true, show URL addresses after external links. @@ -355,7 +361,10 @@ intersphinx_mapping = { "python": ("https://python.readthedocs.org/en/latest/", None), "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), - "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), + "google.api_core": ( + "https://googleapis.dev/python/google-api-core/latest/", + None, + ), "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), diff --git a/google/api_core/datetime_helpers.py b/google/api_core/datetime_helpers.py index 78268efc..9470863a 100644 --- a/google/api_core/datetime_helpers.py +++ b/google/api_core/datetime_helpers.py @@ -151,7 +151,7 @@ def from_rfc3339(value): micros = 0 else: scale = 9 - len(fraction) - nanos = int(fraction) * (10 ** scale) + nanos = int(fraction) * (10**scale) micros = nanos // 1000 return bare_seconds.replace(microsecond=micros, tzinfo=datetime.timezone.utc) @@ -245,7 +245,7 @@ def from_rfc3339(cls, stamp): nanos = 0 else: scale = 9 - len(fraction) - nanos = int(fraction) * (10 ** scale) + nanos = int(fraction) * (10**scale) return cls( bare.year, bare.month, diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 38fe6e78..bd2f8562 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -202,7 +202,7 @@ def details(self): Returns: Sequence[Any]: A list of structured objects from error_details.proto - """ + """ return list(self._details) @property @@ -490,7 +490,9 @@ def from_http_response(response): error_info = error_info[0] if error_info else None message = "{method} {url}: {error}".format( - method=response.request.method, url=response.request.url, error=error_message, + method=response.request.method, + url=response.request.url, + error=error_message, ) exception = from_http_status( diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index 594df987..e86ddde5 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -246,7 +246,9 @@ def _create_composite_credentials( # Create the metadata plugin for inserting the authorization header. metadata_plugin = google.auth.transport.grpc.AuthMetadataPlugin( - credentials, request, default_host=default_host, + credentials, + request, + default_host=default_host, ) # Create a set of grpc.CallCredentials using the metadata plugin. diff --git a/google/api_core/operations_v1/abstract_operations_client.py b/google/api_core/operations_v1/abstract_operations_client.py index 631094e7..e02bc199 100644 --- a/google/api_core/operations_v1/abstract_operations_client.py +++ b/google/api_core/operations_v1/abstract_operations_client.py @@ -49,7 +49,8 @@ class AbstractOperationsClientMeta(type): _transport_registry["rest"] = OperationsRestTransport def get_transport_class( - cls, label: Optional[str] = None, + cls, + label: Optional[str] = None, ) -> Type[OperationsTransport]: """Returns an appropriate transport class. @@ -165,7 +166,9 @@ def transport(self) -> OperationsTransport: return self._transport @staticmethod - def common_billing_account_path(billing_account: str,) -> str: + def common_billing_account_path( + billing_account: str, + ) -> str: """Returns a fully-qualified billing_account string.""" return "billingAccounts/{billing_account}".format( billing_account=billing_account, @@ -178,9 +181,13 @@ def parse_common_billing_account_path(path: str) -> Dict[str, str]: return m.groupdict() if m else {} @staticmethod - def common_folder_path(folder: str,) -> str: + def common_folder_path( + folder: str, + ) -> str: """Returns a fully-qualified folder string.""" - return "folders/{folder}".format(folder=folder,) + return "folders/{folder}".format( + folder=folder, + ) @staticmethod def parse_common_folder_path(path: str) -> Dict[str, str]: @@ -189,9 +196,13 @@ def parse_common_folder_path(path: str) -> Dict[str, str]: return m.groupdict() if m else {} @staticmethod - def common_organization_path(organization: str,) -> str: + def common_organization_path( + organization: str, + ) -> str: """Returns a fully-qualified organization string.""" - return "organizations/{organization}".format(organization=organization,) + return "organizations/{organization}".format( + organization=organization, + ) @staticmethod def parse_common_organization_path(path: str) -> Dict[str, str]: @@ -200,9 +211,13 @@ def parse_common_organization_path(path: str) -> Dict[str, str]: return m.groupdict() if m else {} @staticmethod - def common_project_path(project: str,) -> str: + def common_project_path( + project: str, + ) -> str: """Returns a fully-qualified project string.""" - return "projects/{project}".format(project=project,) + return "projects/{project}".format( + project=project, + ) @staticmethod def parse_common_project_path(path: str) -> Dict[str, str]: @@ -211,10 +226,14 @@ def parse_common_project_path(path: str) -> Dict[str, str]: return m.groupdict() if m else {} @staticmethod - def common_location_path(project: str, location: str,) -> str: + def common_location_path( + project: str, + location: str, + ) -> str: """Returns a fully-qualified location string.""" return "projects/{project}/locations/{location}".format( - project=project, location=location, + project=project, + location=location, ) @staticmethod @@ -406,12 +425,20 @@ def list_operations( ) # Send the request. - response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) # This method is paged; wrap the response in a pager, which provides # an `__iter__` convenience method. response = pagers.ListOperationsPager( - method=rpc, request=request, response=response, metadata=metadata, + method=rpc, + request=request, + response=response, + metadata=metadata, ) # Done; return the response. @@ -459,7 +486,12 @@ def get_operation( ) # Send the request. - response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) # Done; return the response. return response @@ -506,7 +538,10 @@ def delete_operation( # Send the request. rpc( - request, retry=retry, timeout=timeout, metadata=metadata, + request, + retry=retry, + timeout=timeout, + metadata=metadata, ) def cancel_operation( @@ -560,5 +595,8 @@ def cancel_operation( # Send the request. rpc( - request, retry=retry, timeout=timeout, metadata=metadata, + request, + retry=retry, + timeout=timeout, + metadata=metadata, ) diff --git a/google/api_core/operations_v1/transports/base.py b/google/api_core/operations_v1/transports/base.py index 460e6460..f9070567 100644 --- a/google/api_core/operations_v1/transports/base.py +++ b/google/api_core/operations_v1/transports/base.py @@ -183,9 +183,9 @@ def _prep_wrapped_messages(self, client_info): def close(self): """Closes resources associated with the transport. - .. warning:: - Only call this method if the transport is NOT shared - with other clients - this may cause errors in other clients! + .. warning:: + Only call this method if the transport is NOT shared + with other clients - this may cause errors in other clients! """ raise NotImplementedError() diff --git a/google/api_core/path_template.py b/google/api_core/path_template.py index b6284c85..a99a4c81 100644 --- a/google/api_core/path_template.py +++ b/google/api_core/path_template.py @@ -239,26 +239,26 @@ def validate(tmpl, path): def transcode(http_options, **request_kwargs): """Transcodes a grpc request pattern into a proper HTTP request following the rules outlined here, - https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44-L312 - - Args: - http_options (list(dict)): A list of dicts which consist of these keys, - 'method' (str): The http method - 'uri' (str): The path template - 'body' (str): The body field name (optional) - (This is a simplified representation of the proto option `google.api.http`) - - request_kwargs (dict) : A dict representing the request object - - Returns: - dict: The transcoded request with these keys, - 'method' (str) : The http method - 'uri' (str) : The expanded uri - 'body' (dict) : A dict representing the body (optional) - 'query_params' (dict) : A dict mapping query parameter variables and values - - Raises: - ValueError: If the request does not match the given template. + https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44-L312 + + Args: + http_options (list(dict)): A list of dicts which consist of these keys, + 'method' (str): The http method + 'uri' (str): The path template + 'body' (str): The body field name (optional) + (This is a simplified representation of the proto option `google.api.http`) + + request_kwargs (dict) : A dict representing the request object + + Returns: + dict: The transcoded request with these keys, + 'method' (str) : The http method + 'uri' (str) : The expanded uri + 'body' (dict) : A dict representing the body (optional) + 'query_params' (dict) : A dict mapping query parameter variables and values + + Raises: + ValueError: If the request does not match the given template. """ for http_option in http_options: request = {} diff --git a/google/api_core/rest_streaming.py b/google/api_core/rest_streaming.py index 69f5b41b..f91381c1 100644 --- a/google/api_core/rest_streaming.py +++ b/google/api_core/rest_streaming.py @@ -48,8 +48,7 @@ def __init__(self, response: requests.Response, response_message_cls): self._escape_next = False def cancel(self): - """Cancel existing streaming operation. - """ + """Cancel existing streaming operation.""" self._response.close() def _process_chunk(self, chunk: str): diff --git a/noxfile.py b/noxfile.py index e02a1280..c9333219 100644 --- a/noxfile.py +++ b/noxfile.py @@ -21,7 +21,7 @@ import nox # pytype: disable=import-error -BLACK_VERSION = "black==19.10b0" +BLACK_VERSION = "black==22.3.0" BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] # Black and flake8 clash on the syntax for ignoring flake8's F401 in this file. BLACK_EXCLUDES = ["--exclude", "^/google/api_core/operations_v1/__init__.py"] @@ -64,7 +64,10 @@ def lint(session): session.install("flake8", "flake8-import-order", BLACK_VERSION) session.install(".") session.run( - "black", "--check", *BLACK_EXCLUDES, *BLACK_PATHS, + "black", + "--check", + *BLACK_EXCLUDES, + *BLACK_PATHS, ) session.run("flake8", "google", "tests") diff --git a/tests/asyncio/gapic/test_method_async.py b/tests/asyncio/gapic/test_method_async.py index 1410747d..11847da7 100644 --- a/tests/asyncio/gapic/test_method_async.py +++ b/tests/asyncio/gapic/test_method_async.py @@ -162,7 +162,8 @@ async def test_wrap_method_with_default_retry_and_timeout_using_sentinel(unused_ ) result = await wrapped_method( - retry=gapic_v1.method_async.DEFAULT, timeout=gapic_v1.method_async.DEFAULT, + retry=gapic_v1.method_async.DEFAULT, + timeout=gapic_v1.method_async.DEFAULT, ) assert result == 42 diff --git a/tests/unit/operations_v1/test_operations_rest_client.py b/tests/unit/operations_v1/test_operations_rest_client.py index dddf6b71..625539e2 100644 --- a/tests/unit/operations_v1/test_operations_rest_client.py +++ b/tests/unit/operations_v1/test_operations_rest_client.py @@ -399,7 +399,9 @@ def test_operations_client_client_options_scopes( client_class, transport_class, transport_name ): # Check the case scopes are provided. - options = client_options.ClientOptions(scopes=["1", "2"],) + options = client_options.ClientOptions( + scopes=["1", "2"], + ) with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None client = client_class(client_options=options) @@ -513,10 +515,12 @@ def test_list_operations_rest_pager(): next_page_token="abc", ), operations_pb2.ListOperationsResponse( - operations=[], next_page_token="def", + operations=[], + next_page_token="def", ), operations_pb2.ListOperationsResponse( - operations=[operations_pb2.Operation()], next_page_token="ghi", + operations=[operations_pb2.Operation()], + next_page_token="ghi", ), operations_pb2.ListOperationsResponse( operations=[operations_pb2.Operation(), operations_pb2.Operation()], @@ -553,7 +557,9 @@ def test_get_operation_rest( with mock.patch.object(Session, "request") as req: # Designate an appropriate value for the returned response. return_value = operations_pb2.Operation( - name="operations/sample1", done=True, error=status_pb2.Status(code=411), + name="operations/sample1", + done=True, + error=status_pb2.Status(code=411), ) # Wrap the value into a proper Response obj @@ -679,7 +685,8 @@ def test_credentials_transport_error(): ) with pytest.raises(ValueError): AbstractOperationsClient( - credentials=ga_credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, ) # It is an error to provide a credentials file and a transport instance. @@ -698,7 +705,8 @@ def test_credentials_transport_error(): ) with pytest.raises(ValueError): AbstractOperationsClient( - client_options={"scopes": ["1", "2"]}, transport=transport, + client_options={"scopes": ["1", "2"]}, + transport=transport, ) @@ -765,7 +773,8 @@ def test_operations_base_transport_with_credentials_file(): Transport.return_value = None load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) transports.OperationsTransport( - credentials_file="credentials.json", quota_project_id="octopus", + credentials_file="credentials.json", + quota_project_id="octopus", ) load_creds.assert_called_once_with( "credentials.json", @@ -792,7 +801,9 @@ def test_operations_auth_adc(): adc.return_value = (ga_credentials.AnonymousCredentials(), None) AbstractOperationsClient() adc.assert_called_once_with( - scopes=None, default_scopes=(), quota_project_id=None, + scopes=None, + default_scopes=(), + quota_project_id=None, ) @@ -849,7 +860,9 @@ def test_parse_common_billing_account_path(): def test_common_folder_path(): folder = "whelk" - expected = "folders/{folder}".format(folder=folder,) + expected = "folders/{folder}".format( + folder=folder, + ) actual = AbstractOperationsClient.common_folder_path(folder) assert expected == actual @@ -867,7 +880,9 @@ def test_parse_common_folder_path(): def test_common_organization_path(): organization = "oyster" - expected = "organizations/{organization}".format(organization=organization,) + expected = "organizations/{organization}".format( + organization=organization, + ) actual = AbstractOperationsClient.common_organization_path(organization) assert expected == actual @@ -885,7 +900,9 @@ def test_parse_common_organization_path(): def test_common_project_path(): project = "cuttlefish" - expected = "projects/{project}".format(project=project,) + expected = "projects/{project}".format( + project=project, + ) actual = AbstractOperationsClient.common_project_path(project) assert expected == actual @@ -905,7 +922,8 @@ def test_common_location_path(): project = "winkle" location = "nautilus" expected = "projects/{project}/locations/{location}".format( - project=project, location=location, + project=project, + location=location, ) actual = AbstractOperationsClient.common_location_path(project, location) assert expected == actual @@ -930,7 +948,8 @@ def test_client_withDEFAULT_CLIENT_INFO(): transports.OperationsTransport, "_prep_wrapped_messages" ) as prep: AbstractOperationsClient( - credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, ) prep.assert_called_once_with(client_info) @@ -939,6 +958,7 @@ def test_client_withDEFAULT_CLIENT_INFO(): ) as prep: transport_class = AbstractOperationsClient.get_transport_class() transport_class( - credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, ) prep.assert_called_once_with(client_info) diff --git a/tests/unit/test_client_options.py b/tests/unit/test_client_options.py index fbff5457..334b8c1f 100644 --- a/tests/unit/test_client_options.py +++ b/tests/unit/test_client_options.py @@ -98,7 +98,8 @@ def test_constructor_with_api_key(): def test_constructor_with_both_api_key_and_credentials_file(): with pytest.raises(ValueError): client_options.ClientOptions( - api_key="api-key", credentials_file="path/to/credentials.json", + api_key="api-key", + credentials_file="path/to/credentials.json", ) diff --git a/tests/unit/test_extended_operation.py b/tests/unit/test_extended_operation.py index 9521fde8..ad62b778 100644 --- a/tests/unit/test_extended_operation.py +++ b/tests/unit/test_extended_operation.py @@ -59,7 +59,9 @@ def make_extended_operation(responses=None): refresh.responses = client_operations_responses cancel = mock.Mock(spec=["__call__"]) extended_operation_future = extended_operation.ExtendedOperation.make( - refresh, cancel, client_operations_responses[0], + refresh, + cancel, + client_operations_responses[0], ) return extended_operation_future, refresh, cancel diff --git a/tests/unit/test_page_iterator.py b/tests/unit/test_page_iterator.py index a44e998b..cf43aedf 100644 --- a/tests/unit/test_page_iterator.py +++ b/tests/unit/test_page_iterator.py @@ -505,7 +505,8 @@ def api_request(*args, **kw): assert list(next(items_iter)) == [ dict(name=str(i)) for i in range( - ipage * page_size, min((ipage + 1) * page_size, n_results), + ipage * page_size, + min((ipage + 1) * page_size, n_results), ) ] else: diff --git a/tests/unit/test_rest_streaming.py b/tests/unit/test_rest_streaming.py index 4be59580..1b34a04f 100644 --- a/tests/unit/test_rest_streaming.py +++ b/tests/unit/test_rest_streaming.py @@ -83,7 +83,10 @@ def __next__(self): return x.decode("utf-8") def __init__( - self, responses: List[proto.Message], response_cls, random_split=False, + self, + responses: List[proto.Message], + response_cls, + random_split=False, ): super().__init__() self._responses = responses @@ -105,7 +108,8 @@ def close(self): def iter_content(self, *args, **kwargs): return self._ResponseItr( - self._parse_responses(self._responses), random_split=self._random_split, + self._parse_responses(self._responses), + random_split=self._random_split, ) From 7186e36b72dafc821629455254999c914cdca43f Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 1 Apr 2022 00:12:27 +0000 Subject: [PATCH 072/112] chore(python): update .pre-commit-config.yaml to use black==22.3.0 (#363) Source-Link: https://github.com/googleapis/synthtool/commit/7804ade3daae0d66649bee8df6c55484c6580b8d Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:eede5672562a32821444a8e803fb984a6f61f2237ea3de229d2de24453f4ae7d --- .github/.OwlBot.lock.yaml | 3 ++- .pre-commit-config.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 87dd0061..22cc254a 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:7cffbc10910c3ab1b852c05114a08d374c195a81cdec1d4a67a1d129331d0bfe + digest: sha256:eede5672562a32821444a8e803fb984a6f61f2237ea3de229d2de24453f4ae7d +# created: 2022-03-30T23:44:26.560599165Z diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 62eb5a77..46d23716 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: end-of-file-fixer - id: check-yaml - repo: https://github.com/psf/black - rev: 19.10b0 + rev: 22.3.0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 From 45b0b52730938cdb77d3e99dc602d3dcf4917403 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 31 Mar 2022 21:59:34 -0400 Subject: [PATCH 073/112] chore(python): Enable size-label bot (#364) Source-Link: https://github.com/googleapis/synthtool/commit/06e82790dd719a165ad32b8a06f8f6ec3e3cae0f Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:b3500c053313dc34e07b1632ba9e4e589f4f77036a7cf39e1fe8906811ae0fce Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- .github/auto-label.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 .github/auto-label.yaml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 22cc254a..58a0b153 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:eede5672562a32821444a8e803fb984a6f61f2237ea3de229d2de24453f4ae7d -# created: 2022-03-30T23:44:26.560599165Z + digest: sha256:b3500c053313dc34e07b1632ba9e4e589f4f77036a7cf39e1fe8906811ae0fce +# created: 2022-04-01T01:42:03.609279246Z diff --git a/.github/auto-label.yaml b/.github/auto-label.yaml new file mode 100644 index 00000000..09c8d735 --- /dev/null +++ b/.github/auto-label.yaml @@ -0,0 +1,2 @@ +requestsize: + enabled: true From caba5f22b35305531626bafc4e0041619208ca26 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 6 Apr 2022 06:44:25 -0400 Subject: [PATCH 074/112] chore(python): add license header to auto-label.yaml (#365) Source-Link: https://github.com/googleapis/synthtool/commit/eb78c980b52c7c6746d2edb77d9cf7aaa99a2aab Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:8a5d3f6a2e43ed8293f34e06a2f56931d1e88a2694c3bb11b15df4eb256ad163 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- .github/auto-label.yaml | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 58a0b153..bc893c97 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:b3500c053313dc34e07b1632ba9e4e589f4f77036a7cf39e1fe8906811ae0fce -# created: 2022-04-01T01:42:03.609279246Z + digest: sha256:8a5d3f6a2e43ed8293f34e06a2f56931d1e88a2694c3bb11b15df4eb256ad163 +# created: 2022-04-06T10:30:21.687684602Z diff --git a/.github/auto-label.yaml b/.github/auto-label.yaml index 09c8d735..41bff0b5 100644 --- a/.github/auto-label.yaml +++ b/.github/auto-label.yaml @@ -1,2 +1,15 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. requestsize: enabled: true From 523dbd0b10d37ffcf83fa751f0bad313f162abf1 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 13 Apr 2022 12:22:14 -0400 Subject: [PATCH 075/112] fix: remove dependency on pkg_resources (#361) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #360 🦕 --- google/api_core/client_info.py | 8 ++++---- google/api_core/grpc_helpers.py | 9 --------- google/api_core/operations_v1/transports/base.py | 15 +++++---------- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/google/api_core/client_info.py b/google/api_core/client_info.py index e093ffda..3e4376c9 100644 --- a/google/api_core/client_info.py +++ b/google/api_core/client_info.py @@ -21,8 +21,6 @@ import platform from typing import Union -import pkg_resources - from google.api_core import version as api_core_version _PY_VERSION = platform.python_version() @@ -31,8 +29,10 @@ _GRPC_VERSION: Union[str, None] try: - _GRPC_VERSION = pkg_resources.get_distribution("grpcio").version -except pkg_resources.DistributionNotFound: # pragma: NO COVER + import grpc + + _GRPC_VERSION = grpc.__version__ +except ImportError: # pragma: NO COVER _GRPC_VERSION = None diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index e86ddde5..db16f6fc 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -18,7 +18,6 @@ import functools import grpc -import pkg_resources from google.api_core import exceptions import google.auth @@ -33,14 +32,6 @@ except ImportError: HAS_GRPC_GCP = False -try: - # google.auth.__version__ was added in 1.26.0 - _GOOGLE_AUTH_VERSION = google.auth.__version__ -except AttributeError: - try: # try pkg_resources if it is available - _GOOGLE_AUTH_VERSION = pkg_resources.get_distribution("google-auth").version - except pkg_resources.DistributionNotFound: # pragma: NO COVER - _GOOGLE_AUTH_VERSION = None # The list of gRPC Callable interfaces that return iterators. _STREAM_WRAP_CLASSES = (grpc.UnaryStreamMultiCallable, grpc.StreamStreamMultiCallable) diff --git a/google/api_core/operations_v1/transports/base.py b/google/api_core/operations_v1/transports/base.py index f9070567..e19bc3e8 100644 --- a/google/api_core/operations_v1/transports/base.py +++ b/google/api_core/operations_v1/transports/base.py @@ -16,26 +16,21 @@ import abc from typing import Awaitable, Callable, Optional, Sequence, Union -import pkg_resources - import google.api_core # type: ignore from google.api_core import exceptions as core_exceptions # type: ignore from google.api_core import gapic_v1 # type: ignore from google.api_core import retry as retries # type: ignore +from google.api_core import version import google.auth # type: ignore from google.auth import credentials as ga_credentials # type: ignore from google.longrunning import operations_pb2 from google.oauth2 import service_account # type: ignore from google.protobuf import empty_pb2 # type: ignore -try: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( - gapic_version=pkg_resources.get_distribution( - "google.api_core.operations_v1", - ).version, - ) -except pkg_resources.DistributionNotFound: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=version.__version__, +) class OperationsTransport(abc.ABC): From 319c02638682563edc1deb24c5e8ba28d15facab Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 13 Apr 2022 12:26:39 -0400 Subject: [PATCH 076/112] chore(main): release 2.7.2 (#358) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 ++++++++ google/api_core/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0610df01..5ac245f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.7.2](https://github.com/googleapis/python-api-core/compare/v2.7.1...v2.7.2) (2022-04-13) + + +### Bug Fixes + +* allow grpc without grpcio-status ([#355](https://github.com/googleapis/python-api-core/issues/355)) ([112049e](https://github.com/googleapis/python-api-core/commit/112049e79f5a5b0a989d85d438a1bd29485f46f7)) +* remove dependency on pkg_resources ([#361](https://github.com/googleapis/python-api-core/issues/361)) ([523dbd0](https://github.com/googleapis/python-api-core/commit/523dbd0b10d37ffcf83fa751f0bad313f162abf1)) + ### [2.7.1](https://github.com/googleapis/python-api-core/compare/v2.7.0...v2.7.1) (2022-03-09) diff --git a/google/api_core/version.py b/google/api_core/version.py index a97caa56..52126b8e 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.7.1" +__version__ = "2.7.2" From 12f76670f0f0e329be19964a7c1a678c1fe41534 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 21 Apr 2022 17:55:06 +0200 Subject: [PATCH 077/112] chore(deps): update all dependencies (#367) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- .github/workflows/unittest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 1a857a3d..77bfc2cc 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -42,7 +42,7 @@ jobs: run: | nox -s unit${{ matrix.option }}-${{ matrix.python }} - name: Upload coverage results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: coverage-artifacts path: .coverage${{ matrix.option }}-${{ matrix.python }} @@ -64,7 +64,7 @@ jobs: python -m pip install --upgrade setuptools pip wheel python -m pip install coverage - name: Download coverage results - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: coverage-artifacts path: .coverage-results/ From 44978730b9f7fc1e2dcc6a182f282794807f3eae Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 16:26:27 +0000 Subject: [PATCH 078/112] chore(python): use ubuntu 22.04 in docs image (#368) Source-Link: https://github.com/googleapis/synthtool/commit/f15cc72fb401b4861cedebb10af74afe428fb1f8 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:bc5eed3804aec2f05fad42aacf973821d9500c174015341f721a984a0825b6fd --- .github/.OwlBot.lock.yaml | 4 ++-- .kokoro/docker/docs/Dockerfile | 20 ++++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index bc893c97..64f82d6b 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:8a5d3f6a2e43ed8293f34e06a2f56931d1e88a2694c3bb11b15df4eb256ad163 -# created: 2022-04-06T10:30:21.687684602Z + digest: sha256:bc5eed3804aec2f05fad42aacf973821d9500c174015341f721a984a0825b6fd +# created: 2022-04-21T15:43:16.246106921Z diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile index 4e1b1fb8..238b87b9 100644 --- a/.kokoro/docker/docs/Dockerfile +++ b/.kokoro/docker/docs/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ubuntu:20.04 +from ubuntu:22.04 ENV DEBIAN_FRONTEND noninteractive @@ -60,8 +60,24 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -f /var/cache/apt/archives/*.deb +###################### Install python 3.8.11 + +# Download python 3.8.11 +RUN wget https://www.python.org/ftp/python/3.8.11/Python-3.8.11.tgz + +# Extract files +RUN tar -xvf Python-3.8.11.tgz + +# Install python 3.8.11 +RUN ./Python-3.8.11/configure --enable-optimizations +RUN make altinstall + +###################### Install pip RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ - && python3.8 /tmp/get-pip.py \ + && python3 /tmp/get-pip.py \ && rm /tmp/get-pip.py +# Test pip +RUN python3 -m pip + CMD ["python3.8"] From 022add16266f9c07f0f88eea13472cc2e0bfc991 Mon Sep 17 00:00:00 2001 From: Dorian Kind Date: Tue, 26 Apr 2022 12:44:33 +0200 Subject: [PATCH 079/112] fix: Avoid AttributeError if grpcio-status is not installed (#370) --- google/api_core/exceptions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index bd2f8562..aaba8791 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -599,7 +599,9 @@ def from_grpc_error(rpc_exc): """ # NOTE(lidiz) All gRPC error shares the parent class grpc.RpcError. # However, check for grpc.RpcError breaks backward compatibility. - if isinstance(rpc_exc, grpc.Call) or _is_informative_grpc_error(rpc_exc): + if ( + grpc is not None and isinstance(rpc_exc, grpc.Call) + ) or _is_informative_grpc_error(rpc_exc): details, err_info = _parse_grpc_error_details(rpc_exc) return from_grpc_status( rpc_exc.code(), From 64802e104bf47c32a212e00494d4bf0bbcb2fe33 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 26 Apr 2022 18:26:17 -0400 Subject: [PATCH 080/112] test: use `not` instead of `is False` in assert (#366) * test: use == instead of is when comparing equality * use not instead of == --- tests/unit/test_extended_operation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_extended_operation.py b/tests/unit/test_extended_operation.py index ad62b778..c551bfa8 100644 --- a/tests/unit/test_extended_operation.py +++ b/tests/unit/test_extended_operation.py @@ -70,8 +70,8 @@ def make_extended_operation(responses=None): def test_constructor(): ex_op, refresh, _ = make_extended_operation() assert ex_op._extended_operation == refresh.responses[0] - assert ex_op.cancelled() is False - assert ex_op.done() is False + assert not ex_op.cancelled() + assert not ex_op.done() assert ex_op.name == TEST_OPERATION_NAME assert ex_op.status == CustomOperation.StatusCode.PENDING assert ex_op.error_code is None From 1ce2e5409e5b97a146390fcaaa385fe80087d571 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 29 Apr 2022 13:32:45 -0400 Subject: [PATCH 081/112] test: fix KeyError in test_rest_streaming.py (#373) --- tests/unit/test_rest_streaming.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_rest_streaming.py b/tests/unit/test_rest_streaming.py index 1b34a04f..a44c83c0 100644 --- a/tests/unit/test_rest_streaming.py +++ b/tests/unit/test_rest_streaming.py @@ -28,6 +28,7 @@ from google.protobuf import timestamp_pb2 +__protobuf__ = proto.module(package=__name__) SEED = int(time.time()) logging.info(f"Starting rest streaming tests with random seed: {SEED}") random.seed(SEED) From 52366f235bc1b94895d245188c7d71f0d4d1246a Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 2 May 2022 05:24:26 -0400 Subject: [PATCH 082/112] chore(main): release 2.7.3 (#371) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ac245f8..bfebfa51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.7.3](https://github.com/googleapis/python-api-core/compare/v2.7.2...v2.7.3) (2022-04-29) + + +### Bug Fixes + +* Avoid AttributeError if grpcio-status is not installed ([#370](https://github.com/googleapis/python-api-core/issues/370)) ([022add1](https://github.com/googleapis/python-api-core/commit/022add16266f9c07f0f88eea13472cc2e0bfc991)) + ### [2.7.2](https://github.com/googleapis/python-api-core/compare/v2.7.1...v2.7.2) (2022-04-13) diff --git a/google/api_core/version.py b/google/api_core/version.py index 52126b8e..138e01b3 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.7.2" +__version__ = "2.7.3" From 364776ca5256e6202f8eaa0b589c18aaf7ba0680 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 11:44:43 -0400 Subject: [PATCH 083/112] chore: [autoapprove] update readme_gen.py to include autoescape True (#374) Source-Link: https://github.com/googleapis/synthtool/commit/6b4d5a6407d740beb4158b302194a62a4108a8a6 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:f792ee1320e03eda2d13a5281a2989f7ed8a9e50b73ef6da97fac7e1e850b149 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- scripts/readme-gen/readme_gen.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 64f82d6b..b631901e 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:bc5eed3804aec2f05fad42aacf973821d9500c174015341f721a984a0825b6fd -# created: 2022-04-21T15:43:16.246106921Z + digest: sha256:f792ee1320e03eda2d13a5281a2989f7ed8a9e50b73ef6da97fac7e1e850b149 +# created: 2022-05-05T15:17:27.599381182Z diff --git a/scripts/readme-gen/readme_gen.py b/scripts/readme-gen/readme_gen.py index d309d6e9..91b59676 100644 --- a/scripts/readme-gen/readme_gen.py +++ b/scripts/readme-gen/readme_gen.py @@ -28,7 +28,10 @@ jinja_env = jinja2.Environment( trim_blocks=True, loader=jinja2.FileSystemLoader( - os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates')))) + os.path.abspath(os.path.join(os.path.dirname(__file__), "templates")) + ), + autoescape=True, +) README_TMPL = jinja_env.get_template('README.tmpl.rst') From cddc802449b222877aae81c0968e5b6ed87ee14d Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 23:04:25 +0000 Subject: [PATCH 084/112] chore(python): auto approve template changes (#376) Source-Link: https://github.com/googleapis/synthtool/commit/453a5d9c9a55d1969240a37d36cec626d20a9024 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 --- .github/.OwlBot.lock.yaml | 4 ++-- .github/auto-approve.yml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 .github/auto-approve.yml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index b631901e..757c9dca 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:f792ee1320e03eda2d13a5281a2989f7ed8a9e50b73ef6da97fac7e1e850b149 -# created: 2022-05-05T15:17:27.599381182Z + digest: sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 +# created: 2022-05-05T22:08:23.383410683Z diff --git a/.github/auto-approve.yml b/.github/auto-approve.yml new file mode 100644 index 00000000..311ebbb8 --- /dev/null +++ b/.github/auto-approve.yml @@ -0,0 +1,3 @@ +# https://github.com/googleapis/repo-automation-bots/tree/main/packages/auto-approve +processes: + - "OwlBotTemplateChanges" From c97c4980125a86f384cdf12720df7bb1a2adf9d2 Mon Sep 17 00:00:00 2001 From: Aza Tulepbergenov Date: Wed, 18 May 2022 12:03:14 -0700 Subject: [PATCH 085/112] feat: adds support for audience in client_options (#379) feat: adds support for audience in client_options. --- google/api_core/client_options.py | 7 +++++++ tests/unit/test_client_options.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/google/api_core/client_options.py b/google/api_core/client_options.py index 4a2a84a4..ee9f28a9 100644 --- a/google/api_core/client_options.py +++ b/google/api_core/client_options.py @@ -70,6 +70,11 @@ class ClientOptions(object): scopes (Optional[Sequence[str]]): OAuth access token override scopes. api_key (Optional[str]): Google API key. ``credentials_file`` and ``api_key`` are mutually exclusive. + api_audience (Optional[str]): The intended audience for the API calls + to the service that will be set when using certain 3rd party + authentication flows. Audience is typically a resource identifier. + If not set, the service endpoint value will be used as a default. + An example of a valid ``api_audience`` is: "https://language.googleapis.com". Raises: ValueError: If both ``client_cert_source`` and ``client_encrypted_cert_source`` @@ -85,6 +90,7 @@ def __init__( credentials_file=None, scopes=None, api_key=None, + api_audience=None, ): if client_cert_source and client_encrypted_cert_source: raise ValueError( @@ -99,6 +105,7 @@ def __init__( self.credentials_file = credentials_file self.scopes = scopes self.api_key = api_key + self.api_audience = api_audience def __repr__(self): return "ClientOptions: " + repr(self.__dict__) diff --git a/tests/unit/test_client_options.py b/tests/unit/test_client_options.py index 334b8c1f..d56a1b3a 100644 --- a/tests/unit/test_client_options.py +++ b/tests/unit/test_client_options.py @@ -36,6 +36,7 @@ def test_constructor(): "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/cloud-platform.read-only", ], + api_audience="foo2.googleapis.com", ) assert options.api_endpoint == "foo.googleapis.com" @@ -46,6 +47,7 @@ def test_constructor(): "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/cloud-platform.read-only", ] + assert options.api_audience == "foo2.googleapis.com" def test_constructor_with_encrypted_cert_source(): @@ -114,6 +116,7 @@ def test_from_dict(): "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/cloud-platform.read-only", ], + "api_audience": "foo2.googleapis.com", } ) @@ -126,6 +129,7 @@ def test_from_dict(): "https://www.googleapis.com/auth/cloud-platform.read-only", ] assert options.api_key is None + assert options.api_audience == "foo2.googleapis.com" def test_from_dict_bad_argument(): From eed844f211ad8c6ab2c4cb0d6f089e1f11999f71 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 18 May 2022 14:45:58 -0700 Subject: [PATCH 086/112] chore(main): release 2.8.0 (#381) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 ++++++++ google/api_core/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfebfa51..7ed6c616 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.8.0](https://github.com/googleapis/python-api-core/compare/v2.7.3...v2.8.0) (2022-05-18) + + +### Features + +* adds support for audience in client_options ([#379](https://github.com/googleapis/python-api-core/issues/379)) ([c97c498](https://github.com/googleapis/python-api-core/commit/c97c4980125a86f384cdf12720df7bb1a2adf9d2)) +* adds support for audience in client_options. ([c97c498](https://github.com/googleapis/python-api-core/commit/c97c4980125a86f384cdf12720df7bb1a2adf9d2)) + ### [2.7.3](https://github.com/googleapis/python-api-core/compare/v2.7.2...v2.7.3) (2022-04-29) diff --git a/google/api_core/version.py b/google/api_core/version.py index 138e01b3..0a9aecb3 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.7.3" +__version__ = "2.8.0" From d84d66c2a4107f5f9a20c53e870a27fb1250ea3d Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 26 May 2022 16:26:18 -0400 Subject: [PATCH 087/112] fix(deps): require protobuf>= 3.15.0, <4.0.0dev (#385) Also adds upper limits for extras. fix(deps): require googleapis-common-protos >= 1.56.2 --- setup.py | 8 ++++---- testing/constraints-3.6.txt | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index e400a52f..70b26c92 100644 --- a/setup.py +++ b/setup.py @@ -29,15 +29,15 @@ # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 5 - Production/Stable" dependencies = [ - "googleapis-common-protos >= 1.52.0, < 2.0dev", - "protobuf >= 3.12.0", + "googleapis-common-protos >= 1.56.2, < 2.0dev", + "protobuf >= 3.15.0, <4.0.0dev", "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", ] extras = { "grpc": ["grpcio >= 1.33.2, < 2.0dev", "grpcio-status >= 1.33.2, < 2.0dev"], - "grpcgcp": "grpcio-gcp >= 0.2.2", - "grpcio-gcp": "grpcio-gcp >= 0.2.2", + "grpcgcp": "grpcio-gcp >= 0.2.2, < 1.0dev", + "grpcio-gcp": "grpcio-gcp >= 0.2.2, < 1.0dev", } diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index bb1eb1e3..014e2b2a 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -5,8 +5,8 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 -googleapis-common-protos==1.52.0 -protobuf==3.12.0 +googleapis-common-protos==1.56.2 +protobuf==3.15.0 google-auth==1.25.0 requests==2.18.0 packaging==14.3 From 9be727df7fb19d7264c81eab9cc8b6de63f6117e Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 26 May 2022 16:29:43 -0400 Subject: [PATCH 088/112] chore(main): release 2.8.1 (#386) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 ++++++++ google/api_core/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ed6c616..c84458ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://pypi.org/project/google-api-core/#history +### [2.8.1](https://github.com/googleapis/python-api-core/compare/v2.8.0...v2.8.1) (2022-05-26) + + +### Bug Fixes + +* **deps:** require googleapis-common-protos >= 1.56.2 ([d84d66c](https://github.com/googleapis/python-api-core/commit/d84d66c2a4107f5f9a20c53e870a27fb1250ea3d)) +* **deps:** require protobuf>= 3.15.0, <4.0.0dev ([#385](https://github.com/googleapis/python-api-core/issues/385)) ([d84d66c](https://github.com/googleapis/python-api-core/commit/d84d66c2a4107f5f9a20c53e870a27fb1250ea3d)) + ## [2.8.0](https://github.com/googleapis/python-api-core/compare/v2.7.3...v2.8.0) (2022-05-18) diff --git a/google/api_core/version.py b/google/api_core/version.py index 0a9aecb3..7e8f51a6 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.8.0" +__version__ = "2.8.1" From 0c4668df73ac3989170ea5677f906f147a9560d0 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 27 May 2022 12:27:54 -0400 Subject: [PATCH 089/112] chore: allow releases from older version branches (#388) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: allow releases from older version branches * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- .github/release-please.yml | 9 +++++++++ owlbot.py | 2 ++ 2 files changed, 11 insertions(+) diff --git a/.github/release-please.yml b/.github/release-please.yml index 466597e5..29601ad4 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -1,2 +1,11 @@ releaseType: python handleGHRelease: true +# NOTE: this section is generated by synthtool.languages.python +# See https://github.com/googleapis/synthtool/blob/master/synthtool/languages/python.py +branches: +- branch: v1 + handleGHRelease: true + releaseType: python +- branch: v0 + handleGHRelease: true + releaseType: python diff --git a/owlbot.py b/owlbot.py index 0b7f7d21..9d58f4a1 100644 --- a/owlbot.py +++ b/owlbot.py @@ -16,6 +16,7 @@ import synthtool as s from synthtool import gcp +from synthtool.languages import python common = gcp.CommonTemplates() @@ -47,5 +48,6 @@ s.replace(".github/workflows/lint.yml", "python-version: \"3.10\"", "python-version: \"3.7\"") +python.configure_previous_major_version_branches() s.shell.run(["nox", "-s", "blacken"], hide_output=False) From ac266e935bc4e7c6dff250384407e7a60d8dba90 Mon Sep 17 00:00:00 2001 From: Dan Lee <71398022+dandhlee@users.noreply.github.com> Date: Wed, 1 Jun 2022 21:31:15 -0400 Subject: [PATCH 090/112] docs: fix changelog header to consistent size (#394) --- CHANGELOG.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c84458ba..227226de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ [1]: https://pypi.org/project/google-api-core/#history -### [2.8.1](https://github.com/googleapis/python-api-core/compare/v2.8.0...v2.8.1) (2022-05-26) +## [2.8.1](https://github.com/googleapis/python-api-core/compare/v2.8.0...v2.8.1) (2022-05-26) ### Bug Fixes @@ -20,14 +20,14 @@ * adds support for audience in client_options ([#379](https://github.com/googleapis/python-api-core/issues/379)) ([c97c498](https://github.com/googleapis/python-api-core/commit/c97c4980125a86f384cdf12720df7bb1a2adf9d2)) * adds support for audience in client_options. ([c97c498](https://github.com/googleapis/python-api-core/commit/c97c4980125a86f384cdf12720df7bb1a2adf9d2)) -### [2.7.3](https://github.com/googleapis/python-api-core/compare/v2.7.2...v2.7.3) (2022-04-29) +## [2.7.3](https://github.com/googleapis/python-api-core/compare/v2.7.2...v2.7.3) (2022-04-29) ### Bug Fixes * Avoid AttributeError if grpcio-status is not installed ([#370](https://github.com/googleapis/python-api-core/issues/370)) ([022add1](https://github.com/googleapis/python-api-core/commit/022add16266f9c07f0f88eea13472cc2e0bfc991)) -### [2.7.2](https://github.com/googleapis/python-api-core/compare/v2.7.1...v2.7.2) (2022-04-13) +## [2.7.2](https://github.com/googleapis/python-api-core/compare/v2.7.1...v2.7.2) (2022-04-13) ### Bug Fixes @@ -35,7 +35,7 @@ * allow grpc without grpcio-status ([#355](https://github.com/googleapis/python-api-core/issues/355)) ([112049e](https://github.com/googleapis/python-api-core/commit/112049e79f5a5b0a989d85d438a1bd29485f46f7)) * remove dependency on pkg_resources ([#361](https://github.com/googleapis/python-api-core/issues/361)) ([523dbd0](https://github.com/googleapis/python-api-core/commit/523dbd0b10d37ffcf83fa751f0bad313f162abf1)) -### [2.7.1](https://github.com/googleapis/python-api-core/compare/v2.7.0...v2.7.1) (2022-03-09) +## [2.7.1](https://github.com/googleapis/python-api-core/compare/v2.7.0...v2.7.1) (2022-03-09) ### Bug Fixes @@ -49,7 +49,7 @@ * expose extra fields in ExtendedOperation ([#351](https://github.com/googleapis/python-api-core/issues/351)) ([9abc6f4](https://github.com/googleapis/python-api-core/commit/9abc6f48f23c87b9771dca3c96b4f6af39620a50)) -### [2.6.1](https://github.com/googleapis/python-api-core/compare/v2.6.0...v2.6.1) (2022-03-05) +## [2.6.1](https://github.com/googleapis/python-api-core/compare/v2.6.0...v2.6.1) (2022-03-05) ### Bug Fixes @@ -113,14 +113,14 @@ * handle bare 'grpc.Call' in 'from_grpc_error' ([#298](https://www.github.com/googleapis/python-api-core/issues/298)) ([060b339](https://www.github.com/googleapis/python-api-core/commit/060b339e3af296dd1772bfc1b4a0d2b4264cae1f)) -### [2.2.2](https://www.github.com/googleapis/python-api-core/compare/v2.2.1...v2.2.2) (2021-11-02) +## [2.2.2](https://www.github.com/googleapis/python-api-core/compare/v2.2.1...v2.2.2) (2021-11-02) ### Bug Fixes * make 'gapic_v1.method.DEFAULT' a typed object ([#292](https://www.github.com/googleapis/python-api-core/issues/292)) ([ffc51f0](https://www.github.com/googleapis/python-api-core/commit/ffc51f03c7ce5d9f009ba859b8df385d52925578)) -### [2.2.1](https://www.github.com/googleapis/python-api-core/compare/v2.2.0...v2.2.1) (2021-10-26) +## [2.2.1](https://www.github.com/googleapis/python-api-core/compare/v2.2.0...v2.2.1) (2021-10-26) ### Bug Fixes @@ -134,7 +134,7 @@ * add 'GoogleAPICallError.error_details' property ([#286](https://www.github.com/googleapis/python-api-core/issues/286)) ([ef6f0fc](https://www.github.com/googleapis/python-api-core/commit/ef6f0fcfdfe771172056e35e3c990998b3b00416)) -### [2.1.1](https://www.github.com/googleapis/python-api-core/compare/v2.1.0...v2.1.1) (2021-10-13) +## [2.1.1](https://www.github.com/googleapis/python-api-core/compare/v2.1.0...v2.1.1) (2021-10-13) ### Bug Fixes @@ -150,7 +150,7 @@ * Add helper function to format query_params for rest transport. ([#275](https://www.github.com/googleapis/python-api-core/issues/275)) ([1c5eb4d](https://www.github.com/googleapis/python-api-core/commit/1c5eb4df93d78e791082d9282330ebf0faacd222)) * add support for Python 3.10 ([#284](https://www.github.com/googleapis/python-api-core/issues/284)) ([a422a5d](https://www.github.com/googleapis/python-api-core/commit/a422a5d72cb6f363d57e7a4effe421ba8e049cde)) -### [2.0.1](https://www.github.com/googleapis/python-api-core/compare/v2.0.0...v2.0.1) (2021-08-31) +## [2.0.1](https://www.github.com/googleapis/python-api-core/compare/v2.0.0...v2.0.1) (2021-08-31) ### Bug Fixes @@ -179,7 +179,7 @@ * strip trailing _ from field mask paths ([#228](https://www.github.com/googleapis/python-api-core/issues/228)) ([ff6ef1b](https://www.github.com/googleapis/python-api-core/commit/ff6ef1bd07fa68307b7c82c910416d770e7b3416)) -### [1.31.1](https://www.github.com/googleapis/python-api-core/compare/v1.31.0...v1.31.1) (2021-07-26) +## [1.31.1](https://www.github.com/googleapis/python-api-core/compare/v1.31.0...v1.31.1) (2021-07-26) ### Bug Fixes @@ -242,7 +242,7 @@ * Add support for `rest/` token in `x-goog-api-client` header ([#189](https://www.github.com/googleapis/python-api-core/issues/189)) ([15aca6b](https://www.github.com/googleapis/python-api-core/commit/15aca6b288b2ec5ce0251e442e1dfa7f52e1b124)) * retry google.auth TransportError and requests ConnectionError ([#178](https://www.github.com/googleapis/python-api-core/issues/178)) ([6ae04a8](https://www.github.com/googleapis/python-api-core/commit/6ae04a8d134fffe13f06081e15f9723c1b2ea334)) -### [1.26.3](https://www.github.com/googleapis/python-api-core/compare/v1.26.2...v1.26.3) (2021-03-25) +## [1.26.3](https://www.github.com/googleapis/python-api-core/compare/v1.26.2...v1.26.3) (2021-03-25) ### Bug Fixes @@ -254,14 +254,14 @@ * update python contributing guide ([#147](https://www.github.com/googleapis/python-api-core/issues/147)) ([1d76b57](https://www.github.com/googleapis/python-api-core/commit/1d76b57d1f218f7885f85dc7c052bad1ad3857ac)) -### [1.26.2](https://www.github.com/googleapis/python-api-core/compare/v1.26.1...v1.26.2) (2021-03-23) +## [1.26.2](https://www.github.com/googleapis/python-api-core/compare/v1.26.1...v1.26.2) (2021-03-23) ### Bug Fixes * save empty IAM policy bindings ([#155](https://www.github.com/googleapis/python-api-core/issues/155)) ([536c2ca](https://www.github.com/googleapis/python-api-core/commit/536c2cad814b8fa8cd346a3d7bd5f6b9889c4a6f)) -### [1.26.1](https://www.github.com/googleapis/python-api-core/compare/v1.26.0...v1.26.1) (2021-02-12) +## [1.26.1](https://www.github.com/googleapis/python-api-core/compare/v1.26.0...v1.26.1) (2021-02-12) ### Bug Fixes @@ -275,7 +275,7 @@ * allow default_host and default_scopes to be passed to create_channel ([#134](https://www.github.com/googleapis/python-api-core/issues/134)) ([94c76e0](https://www.github.com/googleapis/python-api-core/commit/94c76e0873e5b2f42331d5b1ad286c1e63b61395)) -### [1.25.1](https://www.github.com/googleapis/python-api-core/compare/v1.25.0...v1.25.1) (2021-01-25) +## [1.25.1](https://www.github.com/googleapis/python-api-core/compare/v1.25.0...v1.25.1) (2021-01-25) ### Bug Fixes @@ -299,7 +299,7 @@ * **python:** document adding Python 3.9 support, dropping 3.5 support ([#120](https://www.github.com/googleapis/python-api-core/issues/120)) ([b51b7f5](https://www.github.com/googleapis/python-api-core/commit/b51b7f587042fe9340371c1b5c8e9adf8001c43a)), closes [#787](https://www.github.com/googleapis/python-api-core/issues/787) -### [1.24.1](https://www.github.com/googleapis/python-api-core/compare/v1.24.0...v1.24.1) (2020-12-16) +## [1.24.1](https://www.github.com/googleapis/python-api-core/compare/v1.24.0...v1.24.1) (2020-12-16) ### Bug Fixes @@ -332,28 +332,28 @@ * harden install to use full paths, and windows separators on windows ([#88](https://www.github.com/googleapis/python-api-core/issues/88)) ([db8e636](https://www.github.com/googleapis/python-api-core/commit/db8e636f545a8872f959e3f403cfec30ffed6c34)) * update out-of-date comment in exceptions.py ([#93](https://www.github.com/googleapis/python-api-core/issues/93)) ([70ebe42](https://www.github.com/googleapis/python-api-core/commit/70ebe42601b3d088b3421233ef7d8245229b7265)) -### [1.22.4](https://www.github.com/googleapis/python-api-core/compare/v1.22.3...v1.22.4) (2020-10-05) +## [1.22.4](https://www.github.com/googleapis/python-api-core/compare/v1.22.3...v1.22.4) (2020-10-05) ### Bug Fixes * use version.py instead of pkg_resources.get_distribution ([#80](https://www.github.com/googleapis/python-api-core/issues/80)) ([d480d97](https://www.github.com/googleapis/python-api-core/commit/d480d97e41cd6705325b3b649360553a83c23f47)) -### [1.22.3](https://www.github.com/googleapis/python-api-core/compare/v1.22.2...v1.22.3) (2020-10-02) +## [1.22.3](https://www.github.com/googleapis/python-api-core/compare/v1.22.2...v1.22.3) (2020-10-02) ### Bug Fixes * **deps:** require six >= 1.13.0 ([#78](https://www.github.com/googleapis/python-api-core/issues/78)) ([a7a8b98](https://www.github.com/googleapis/python-api-core/commit/a7a8b98602a3eb277fdc607ac69f3bcb147f3351)), closes [/github.com/benjaminp/six/blob/c0be8815d13df45b6ae471c4c436cce8c192245d/CHANGES#L30-L31](https://www.github.com/googleapis//github.com/benjaminp/six/blob/c0be8815d13df45b6ae471c4c436cce8c192245d/CHANGES/issues/L30-L31) -### [1.22.2](https://www.github.com/googleapis/python-api-core/compare/v1.22.1...v1.22.2) (2020-09-03) +## [1.22.2](https://www.github.com/googleapis/python-api-core/compare/v1.22.1...v1.22.2) (2020-09-03) ### Bug Fixes * only add quota project id if supported ([#75](https://www.github.com/googleapis/python-api-core/issues/75)) ([8f8ee78](https://www.github.com/googleapis/python-api-core/commit/8f8ee7879e4f834f3c676e535ffc41b5b9b2de62)) -### [1.22.1](https://www.github.com/googleapis/python-api-core/compare/v1.22.0...v1.22.1) (2020-08-12) +## [1.22.1](https://www.github.com/googleapis/python-api-core/compare/v1.22.0...v1.22.1) (2020-08-12) ### Documentation @@ -384,7 +384,7 @@ * allow credentials files to be passed for channel creation ([#50](https://www.github.com/googleapis/python-api-core/issues/50)) ([ded92d0](https://www.github.com/googleapis/python-api-core/commit/ded92d0acdcde4295d0e5df05fda0d83783a3991)) -### [1.20.1](https://www.github.com/googleapis/python-api-core/compare/v1.20.0...v1.20.1) (2020-06-16) +## [1.20.1](https://www.github.com/googleapis/python-api-core/compare/v1.20.0...v1.20.1) (2020-06-16) ### Bug Fixes @@ -398,7 +398,7 @@ * allow disabling response stream pre-fetch ([#30](https://www.github.com/googleapis/python-api-core/issues/30)) ([74e0b0f](https://www.github.com/googleapis/python-api-core/commit/74e0b0f8387207933c120af15b2bb5d175dd8f84)), closes [#25](https://www.github.com/googleapis/python-api-core/issues/25) -### [1.19.1](https://www.github.com/googleapis/python-api-core/compare/v1.19.0...v1.19.1) (2020-06-06) +## [1.19.1](https://www.github.com/googleapis/python-api-core/compare/v1.19.0...v1.19.1) (2020-06-06) ### Bug Fixes From 0eb727f92314db3c4383754514f75a49ba02e27b Mon Sep 17 00:00:00 2001 From: Mariatta Wijaya Date: Wed, 1 Jun 2022 18:36:11 -0700 Subject: [PATCH 091/112] docs: Fix typo in the BackgroundConsumer docstring (#395) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `UNAVAILABLE` instead of `UNVAILABLE` Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-api-core/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes # 🦕 --- google/api_core/bidi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index 4b4963f7..57f5f9dd 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -364,7 +364,7 @@ class ResumableBidiRpc(BidiRpc): def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNVAILABLE) + exc.code() == grpc.StatusCode.UNAVAILABLE) initial_request = example_pb2.StreamingRpcRequest( setting='example') @@ -589,7 +589,7 @@ class BackgroundConsumer(object): def should_recover(exc): return ( isinstance(exc, grpc.RpcError) and - exc.code() == grpc.StatusCode.UNVAILABLE) + exc.code() == grpc.StatusCode.UNAVAILABLE) initial_request = example_pb2.StreamingRpcRequest( setting='example') From e92045b7a34b8d0a6374d6b1f67c1c6095ad33c6 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 3 Jun 2022 14:36:25 -0400 Subject: [PATCH 092/112] chore: test minimum dependencies in python 3.7 (#397) * chore: test minimum dependencies in python 3.7 * remove duplicate entry --- testing/constraints-3.7.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index e69de29b..36643dbc 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -0,0 +1,15 @@ +# This constraints file is used to check that lower bounds +# are correct in setup.py +# List *all* library dependencies and extras in this file. +# Pin the version to the lower bound. +# +# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", +# Then this file should have foo==1.14.0 +googleapis-common-protos==1.56.2 +protobuf==3.15.0 +google-auth==1.25.0 +requests==2.18.0 +packaging==14.3 +grpcio==1.33.2 +grpcio-gcp==0.2.2 +grpcio-status==1.33.2 From 5da6733a475c436efc11b14889af73b3a0e20379 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 13 Jun 2022 14:55:32 -0400 Subject: [PATCH 093/112] fix: drop support for grpc-gcp (#401) --- .github/sync-repo-settings.yaml | 5 -- .github/workflows/unittest.yml | 2 +- google/api_core/grpc_helpers.py | 17 +---- google/api_core/grpc_helpers_async.py | 3 - noxfile.py | 19 +----- setup.py | 2 - testing/constraints-3.6.txt | 2 - testing/constraints-3.7.txt | 1 - tests/asyncio/test_grpc_helpers_async.py | 5 +- tests/unit/test_grpc_helpers.py | 80 ++++-------------------- 10 files changed, 20 insertions(+), 116 deletions(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 49c96ed4..b06a42ac 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -16,11 +16,6 @@ branchProtectionRules: - 'unit-3.8' - 'unit-3.9' - 'unit-3.10' - - 'unit_grpc_gcp-3.6' - - 'unit_grpc_gcp-3.7' - - 'unit_grpc_gcp-3.8' - - 'unit_grpc_gcp-3.9' - - 'unit_grpc_gcp-3.10' - 'unit_wo_grpc-3.6' - 'unit_wo_grpc-3.10' - 'cover' diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 77bfc2cc..0c2e37f6 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - option: ["", "_grpc_gcp", "_wo_grpc"] + option: ["", "_wo_grpc"] python: - "3.6" - "3.7" diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index db16f6fc..47d27726 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -25,13 +25,6 @@ import google.auth.transport.grpc import google.auth.transport.requests -try: - import grpc_gcp - - HAS_GRPC_GCP = True -except ImportError: - HAS_GRPC_GCP = False - # The list of gRPC Callable interfaces that return iterators. _STREAM_WRAP_CLASSES = (grpc.UnaryStreamMultiCallable, grpc.StreamStreamMultiCallable) @@ -282,8 +275,7 @@ def create_channel( default_scopes (Sequence[str]): Default scopes passed by a Google client library. Use 'scopes' for user-defined scopes. default_host (str): The default endpoint. e.g., "pubsub.googleapis.com". - kwargs: Additional key-word args passed to - :func:`grpc_gcp.secure_channel` or :func:`grpc.secure_channel`. + kwargs: Additional key-word args passed to :func:`grpc.secure_channel`. Returns: grpc.Channel: The created channel. @@ -302,12 +294,7 @@ def create_channel( default_host=default_host, ) - if HAS_GRPC_GCP: - # If grpc_gcp module is available use grpc_gcp.secure_channel, - # otherwise, use grpc.secure_channel to create grpc channel. - return grpc_gcp.secure_channel(target, composite_credentials, **kwargs) - else: - return grpc.secure_channel(target, composite_credentials, **kwargs) + return grpc.secure_channel(target, composite_credentials, **kwargs) _MethodCall = collections.namedtuple( diff --git a/google/api_core/grpc_helpers_async.py b/google/api_core/grpc_helpers_async.py index 452e7871..63a3ad18 100644 --- a/google/api_core/grpc_helpers_async.py +++ b/google/api_core/grpc_helpers_async.py @@ -27,9 +27,6 @@ from google.api_core import exceptions, grpc_helpers -# TODO(lidiz) Support gRPC GCP wrapper -HAS_GRPC_GCP = False - # NOTE(lidiz) Alternatively, we can hack "__getattribute__" to perform # automatic patching for us. But that means the overhead of creating an # extra Python function spreads to every single send and receive. diff --git a/noxfile.py b/noxfile.py index c9333219..9b71610c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -32,7 +32,6 @@ # 'docfx' is excluded since it only needs to run in 'docs-presubmit' nox.options.sessions = [ "unit", - "unit_grpc_gcp", "unit_wo_grpc", "cover", "pytype", @@ -143,18 +142,6 @@ def unit(session): default(session) -@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) -def unit_grpc_gcp(session): - """Run the unit test suite with grpcio-gcp installed.""" - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" - ) - # Install grpcio-gcp - session.install("-e", ".[grpcgcp]", "-c", constraints_path) - - default(session) - - @nox.session(python=["3.6", "3.10"]) def unit_wo_grpc(session): """Run the unit test suite w/o grpcio installed""" @@ -173,14 +160,14 @@ def lint_setup_py(session): @nox.session(python="3.6") def pytype(session): """Run type-checking.""" - session.install(".[grpc, grpcgcp]", "pytype >= 2019.3.21") + session.install(".[grpc]", "pytype >= 2019.3.21") session.run("pytype") @nox.session(python=DEFAULT_PYTHON_VERSION) def mypy(session): """Run type-checking.""" - session.install(".[grpc, grpcgcp]", "mypy") + session.install(".[grpc]", "mypy") session.install( "types-setuptools", "types-requests", @@ -207,7 +194,7 @@ def cover(session): def docs(session): """Build the docs for this library.""" - session.install("-e", ".[grpc, grpcgcp]") + session.install("-e", ".[grpc]") session.install("sphinx==4.0.1", "alabaster", "recommonmark") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) diff --git a/setup.py b/setup.py index 70b26c92..49bb1034 100644 --- a/setup.py +++ b/setup.py @@ -36,8 +36,6 @@ ] extras = { "grpc": ["grpcio >= 1.33.2, < 2.0dev", "grpcio-status >= 1.33.2, < 2.0dev"], - "grpcgcp": "grpcio-gcp >= 0.2.2, < 1.0dev", - "grpcio-gcp": "grpcio-gcp >= 0.2.2, < 1.0dev", } diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 014e2b2a..c3e6ad74 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -11,6 +11,4 @@ google-auth==1.25.0 requests==2.18.0 packaging==14.3 grpcio==1.33.2 -grpcio-gcp==0.2.2 -grpcio-gcp==0.2.2 grpcio-status==1.33.2 diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 36643dbc..c3e6ad74 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -11,5 +11,4 @@ google-auth==1.25.0 requests==2.18.0 packaging==14.3 grpcio==1.33.2 -grpcio-gcp==0.2.2 grpcio-status==1.33.2 diff --git a/tests/asyncio/test_grpc_helpers_async.py b/tests/asyncio/test_grpc_helpers_async.py index 3681a40d..2d0a1bcd 100644 --- a/tests/asyncio/test_grpc_helpers_async.py +++ b/tests/asyncio/test_grpc_helpers_async.py @@ -565,11 +565,8 @@ def test_create_channel_with_credentials_file_and_default_scopes( grpc_secure_channel.assert_called_once_with(target, composite_creds) -@pytest.mark.skipif( - grpc_helpers_async.HAS_GRPC_GCP, reason="grpc_gcp module not available" -) @mock.patch("grpc.aio.secure_channel") -def test_create_channel_without_grpc_gcp(grpc_secure_channel): +def test_create_channel(grpc_secure_channel): target = "example.com:443" scopes = ["test_scope"] diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py index ca969e48..649072f0 100644 --- a/tests/unit/test_grpc_helpers.py +++ b/tests/unit/test_grpc_helpers.py @@ -365,10 +365,7 @@ def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_c default.assert_called_once_with(scopes=None, default_scopes=None) - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("google.auth.transport.grpc.AuthMetadataPlugin", autospec=True) @@ -400,10 +397,7 @@ def test_create_channel_implicit_with_default_host( mock.sentinel.credentials, mock.sentinel.Request, default_host=default_host ) - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -426,10 +420,7 @@ def test_create_channel_implicit_with_ssl_creds( composite_creds_call.assert_called_once_with(ssl_creds, mock.ANY) composite_creds = composite_creds_call.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -451,10 +442,7 @@ def test_create_channel_implicit_with_scopes( default.assert_called_once_with(scopes=["one", "two"], default_scopes=None) - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -476,10 +464,7 @@ def test_create_channel_implicit_with_default_scopes( default.assert_called_once_with(scopes=None, default_scopes=["three", "four"]) - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) def test_create_channel_explicit_with_duplicate_credentials(): @@ -507,10 +492,7 @@ def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_cred ) assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -530,10 +512,7 @@ def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_cal credentials.with_scopes.assert_called_once_with(scopes, default_scopes=None) assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -557,10 +536,7 @@ def test_create_channel_explicit_default_scopes( ) assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -582,10 +558,7 @@ def test_create_channel_explicit_with_quota_project( credentials.with_quota_project.assert_called_once_with("project-foo") assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -610,10 +583,7 @@ def test_create_channel_with_credentials_file( ) assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -641,10 +611,7 @@ def test_create_channel_with_credentials_file_and_scopes( ) assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -672,32 +639,11 @@ def test_create_channel_with_credentials_file_and_default_scopes( ) assert channel is grpc_secure_channel.return_value - if grpc_helpers.HAS_GRPC_GCP: - grpc_secure_channel.assert_called_once_with(target, composite_creds, None) - else: - grpc_secure_channel.assert_called_once_with(target, composite_creds) + grpc_secure_channel.assert_called_once_with(target, composite_creds) -@pytest.mark.skipif( - not grpc_helpers.HAS_GRPC_GCP, reason="grpc_gcp module not available" -) -@mock.patch("grpc_gcp.secure_channel") -def test_create_channel_with_grpc_gcp(grpc_gcp_secure_channel): - target = "example.com:443" - scopes = ["test_scope"] - - credentials = mock.create_autospec(google.auth.credentials.Scoped, instance=True) - credentials.requires_scopes = True - - grpc_helpers.create_channel(target, credentials=credentials, scopes=scopes) - grpc_gcp_secure_channel.assert_called() - - credentials.with_scopes.assert_called_once_with(scopes, default_scopes=None) - - -@pytest.mark.skipif(grpc_helpers.HAS_GRPC_GCP, reason="grpc_gcp module not available") @mock.patch("grpc.secure_channel") -def test_create_channel_without_grpc_gcp(grpc_secure_channel): +def test_create_channel(grpc_secure_channel): target = "example.com:443" scopes = ["test_scope"] From 8f73d2ee2d3af2201f877aa7e2f7361147759dc7 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 13 Jun 2022 18:10:35 -0400 Subject: [PATCH 094/112] fix(deps): allow protobuf < 5.0.0 (#400) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 49bb1034..3c40548d 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ release_status = "Development Status :: 5 - Production/Stable" dependencies = [ "googleapis-common-protos >= 1.56.2, < 2.0dev", - "protobuf >= 3.15.0, <4.0.0dev", + "protobuf >= 3.15.0, <5.0.0dev", "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", ] From 5b5e77563229687c901d77b5fdecc18168b535e6 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 14 Jun 2022 11:39:20 -0400 Subject: [PATCH 095/112] chore(main): release 2.8.2 (#396) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 14 ++++++++++++++ google/api_core/version.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 227226de..16e22ab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.8.2](https://github.com/googleapis/python-api-core/compare/v2.8.1...v2.8.2) (2022-06-13) + + +### Bug Fixes + +* **deps:** allow protobuf < 5.0.0 ([#400](https://github.com/googleapis/python-api-core/issues/400)) ([8f73d2e](https://github.com/googleapis/python-api-core/commit/8f73d2ee2d3af2201f877aa7e2f7361147759dc7)) +* drop support for grpc-gcp ([#401](https://github.com/googleapis/python-api-core/issues/401)) ([5da6733](https://github.com/googleapis/python-api-core/commit/5da6733a475c436efc11b14889af73b3a0e20379)) + + +### Documentation + +* fix changelog header to consistent size ([#394](https://github.com/googleapis/python-api-core/issues/394)) ([ac266e9](https://github.com/googleapis/python-api-core/commit/ac266e935bc4e7c6dff250384407e7a60d8dba90)) +* Fix typo in the BackgroundConsumer docstring ([#395](https://github.com/googleapis/python-api-core/issues/395)) ([0eb727f](https://github.com/googleapis/python-api-core/commit/0eb727f92314db3c4383754514f75a49ba02e27b)) + ## [2.8.1](https://github.com/googleapis/python-api-core/compare/v2.8.0...v2.8.1) (2022-05-26) diff --git a/google/api_core/version.py b/google/api_core/version.py index 7e8f51a6..839a77a1 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.8.1" +__version__ = "2.8.2" From 7ddb8c00e6be7ab6905a9a802ad1c3063fbfa46c Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 9 Aug 2022 21:44:37 -0400 Subject: [PATCH 096/112] fix: require python 3.7+ (#410) * chore(python): drop python 3.6 Source-Link: https://github.com/googleapis/synthtool/commit/4f89b13af10d086458f9b379e56a614f9d6dab7b Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:e7bb19d47c13839fe8c147e50e02e8b6cf5da8edd1af8b82208cd6f66cc2829c * require python 3.7+ in setup.py * remove python 3.6 sample configs * exclude templated README * remove python 3.6 from noxfile * remove python 3.6 from remaining files * remove testing/constraints-3.6.txt Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 4 +- .github/sync-repo-settings.yaml | 2 - .github/workflows/unittest.yml | 1 - .kokoro/continuous/prerelease-deps.cfg | 7 ++++ .kokoro/presubmit/prerelease-deps.cfg | 7 ++++ .kokoro/samples/python3.6/common.cfg | 40 ------------------- .kokoro/samples/python3.6/continuous.cfg | 7 ---- .kokoro/samples/python3.6/periodic-head.cfg | 11 ----- .kokoro/samples/python3.6/periodic.cfg | 6 --- .kokoro/samples/python3.6/presubmit.cfg | 6 --- .kokoro/test-samples-impl.sh | 4 +- CONTRIBUTING.rst | 16 +------- README.rst | 7 +++- google/api_core/grpc_helpers_async.py | 2 +- mypy.ini | 2 +- noxfile.py | 19 +++++---- owlbot.py | 1 + .../templates/install_deps.tmpl.rst | 2 +- setup.cfg | 2 +- setup.py | 3 +- testing/constraints-3.6.txt | 14 ------- 21 files changed, 39 insertions(+), 124 deletions(-) create mode 100644 .kokoro/continuous/prerelease-deps.cfg create mode 100644 .kokoro/presubmit/prerelease-deps.cfg delete mode 100644 .kokoro/samples/python3.6/common.cfg delete mode 100644 .kokoro/samples/python3.6/continuous.cfg delete mode 100644 .kokoro/samples/python3.6/periodic-head.cfg delete mode 100644 .kokoro/samples/python3.6/periodic.cfg delete mode 100644 .kokoro/samples/python3.6/presubmit.cfg delete mode 100644 testing/constraints-3.6.txt diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 757c9dca..1ce60852 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 -# created: 2022-05-05T22:08:23.383410683Z + digest: sha256:e7bb19d47c13839fe8c147e50e02e8b6cf5da8edd1af8b82208cd6f66cc2829c +# created: 2022-07-05T18:31:20.838186805Z diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index b06a42ac..e2d70f90 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -11,12 +11,10 @@ branchProtectionRules: # No Kokoro: the following are Github actions - 'lint' - 'mypy' - - 'unit-3.6' - 'unit-3.7' - 'unit-3.8' - 'unit-3.9' - 'unit-3.10' - - 'unit_wo_grpc-3.6' - 'unit_wo_grpc-3.10' - 'cover' - 'docs' diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 0c2e37f6..4c1e8c4e 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -13,7 +13,6 @@ jobs: matrix: option: ["", "_wo_grpc"] python: - - "3.6" - "3.7" - "3.8" - "3.9" diff --git a/.kokoro/continuous/prerelease-deps.cfg b/.kokoro/continuous/prerelease-deps.cfg new file mode 100644 index 00000000..3595fb43 --- /dev/null +++ b/.kokoro/continuous/prerelease-deps.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} diff --git a/.kokoro/presubmit/prerelease-deps.cfg b/.kokoro/presubmit/prerelease-deps.cfg new file mode 100644 index 00000000..3595fb43 --- /dev/null +++ b/.kokoro/presubmit/prerelease-deps.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} diff --git a/.kokoro/samples/python3.6/common.cfg b/.kokoro/samples/python3.6/common.cfg deleted file mode 100644 index 3bb6b3a6..00000000 --- a/.kokoro/samples/python3.6/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.6" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-py36" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-api-core/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-api-core/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/continuous.cfg b/.kokoro/samples/python3.6/continuous.cfg deleted file mode 100644 index 7218af14..00000000 --- a/.kokoro/samples/python3.6/continuous.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - diff --git a/.kokoro/samples/python3.6/periodic-head.cfg b/.kokoro/samples/python3.6/periodic-head.cfg deleted file mode 100644 index a18c0cfc..00000000 --- a/.kokoro/samples/python3.6/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-api-core/.kokoro/test-samples-against-head.sh" -} diff --git a/.kokoro/samples/python3.6/periodic.cfg b/.kokoro/samples/python3.6/periodic.cfg deleted file mode 100644 index 71cd1e59..00000000 --- a/.kokoro/samples/python3.6/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/.kokoro/samples/python3.6/presubmit.cfg b/.kokoro/samples/python3.6/presubmit.cfg deleted file mode 100644 index a1c8d975..00000000 --- a/.kokoro/samples/python3.6/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 8a324c9c..2c6500ca 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -33,7 +33,7 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Install nox -python3.6 -m pip install --upgrade --quiet nox +python3.9 -m pip install --upgrade --quiet nox # Use secrets acessor service account to get secrets if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then @@ -76,7 +76,7 @@ for file in samples/**/requirements.txt; do echo "------------------------------------------------------------" # Use nox to execute the tests for the project. - python3.6 -m nox -s "$RUN_TESTS_SESSION" + python3.9 -m nox -s "$RUN_TESTS_SESSION" EXIT=$? # If this is a periodic build, send the test log to the FlakyBot. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6b375f03..dddeddb9 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -21,7 +21,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.6, 3.7, 3.8, 3.9, and 3.10 on both UNIX and Windows. + 3.7, 3.8, 3.9, and 3.10 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -197,13 +197,11 @@ Supported Python Versions We support: -- `Python 3.6`_ - `Python 3.7`_ - `Python 3.8`_ - `Python 3.9`_ - `Python 3.10`_ -.. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ @@ -215,18 +213,6 @@ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-api-core/blob/main/noxfile.py -We also explicitly decided to support Python 3 beginning with version 3.6. -Reasons for this include: - -- Encouraging use of newest versions of Python 3 -- Taking the lead of `prominent`_ open-source `projects`_ -- `Unicode literal support`_ which allows for a cleaner codebase that - works in both Python 2 and Python 3 - -.. _prominent: https://docs.djangoproject.com/en/1.9/faq/install/#what-python-version-can-i-use-with-django -.. _projects: http://flask.pocoo.org/docs/0.10/python3/ -.. _Unicode literal support: https://www.python.org/dev/peps/pep-0414/ - ********** Versioning ********** diff --git a/README.rst b/README.rst index 13d8d3a6..58ae26cb 100644 --- a/README.rst +++ b/README.rst @@ -16,13 +16,16 @@ common helpers used by all Google API clients. For more information, see the Supported Python Versions ------------------------- -Python >= 3.6 +Python >= 3.7 Unsupported Python Versions --------------------------- -Python == 2.7, Python == 3.5. +Python == 2.7, Python == 3.5, Python == 3.6. The last version of this library compatible with Python 2.7 and 3.5 is `google-api-core==1.31.1`. + +The last version of this library compatible with Python 3.6 is +`google-api-core==2.8.2`. diff --git a/google/api_core/grpc_helpers_async.py b/google/api_core/grpc_helpers_async.py index 63a3ad18..5a5bf2a6 100644 --- a/google/api_core/grpc_helpers_async.py +++ b/google/api_core/grpc_helpers_async.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""AsyncIO helpers for :mod:`grpc` supporting 3.6+. +"""AsyncIO helpers for :mod:`grpc` supporting 3.7+. Please combine more detailed docstring in grpc_helpers.py to use following functions. This module is implementing the same surface with AsyncIO semantics. diff --git a/mypy.ini b/mypy.ini index 5c111571..ce33582a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,4 @@ [mypy] -python_version = 3.6 +python_version = 3.7 namespace_packages = True ignore_missing_imports = True diff --git a/noxfile.py b/noxfile.py index 9b71610c..d0db9914 100644 --- a/noxfile.py +++ b/noxfile.py @@ -43,14 +43,14 @@ ] -def _greater_or_equal_than_36(version_string): +def _greater_or_equal_than_37(version_string): tokens = version_string.split(".") for i, token in enumerate(tokens): try: tokens[i] = int(token) except ValueError: pass - return tokens >= [3, 6] + return tokens >= [3, 7] @nox.session(python=DEFAULT_PYTHON_VERSION) @@ -122,9 +122,9 @@ def default(session, install_grpc=True): ), ] - # Inject AsyncIO content and proto-plus, if version >= 3.6. + # Inject AsyncIO content and proto-plus, if version >= 3.7. # proto-plus is needed for a field mask test in test_protobuf_helpers.py - if _greater_or_equal_than_36(session.python): + if _greater_or_equal_than_37(session.python): session.install("asyncmock", "pytest-asyncio", "proto-plus") # Having positional arguments means the user wants to run specific tests. @@ -136,19 +136,19 @@ def default(session, install_grpc=True): session.run(*pytest_args) -@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) +@nox.session(python=["3.7", "3.8", "3.9", "3.10"]) def unit(session): """Run the unit test suite.""" default(session) -@nox.session(python=["3.6", "3.10"]) +@nox.session(python=["3.8", "3.10"]) def unit_wo_grpc(session): """Run the unit test suite w/o grpcio installed""" default(session, install_grpc=False) -@nox.session(python="3.6") +@nox.session(python="3.8") def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" @@ -156,8 +156,7 @@ def lint_setup_py(session): session.run("python", "setup.py", "check", "--restructuredtext", "--strict") -# No 3.7 because pytype supports up to 3.6 only. -@nox.session(python="3.6") +@nox.session(python="3.8") def pytype(session): """Run type-checking.""" session.install(".[grpc]", "pytype >= 2019.3.21") @@ -178,7 +177,7 @@ def mypy(session): session.run("mypy", "google", "tests") -@nox.session(python="3.6") +@nox.session(python="3.8") def cover(session): """Run the final coverage report. diff --git a/owlbot.py b/owlbot.py index 9d58f4a1..ab4f4f0a 100644 --- a/owlbot.py +++ b/owlbot.py @@ -30,6 +30,7 @@ ".coveragerc", # layout "CONTRIBUTING.rst", # no systests ".github/workflows/unittest.yml", # exclude unittest gh action + "README.rst", ] templated_files = common.py_library(microgenerator=True, cov_level=100) s.move(templated_files, excludes=excludes) diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst index 275d6498..6f069c6c 100644 --- a/scripts/readme-gen/templates/install_deps.tmpl.rst +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -12,7 +12,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 3.6+. +#. Create a virtualenv. Samples are compatible with Python 3.7+. .. code-block:: bash diff --git a/setup.cfg b/setup.cfg index 8857e9b0..f7b5a3bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [pytype] -python_version = 3.6 +python_version = 3.7 inputs = google/ exclude = diff --git a/setup.py b/setup.py index 3c40548d..4919e5f8 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,6 @@ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -93,7 +92,7 @@ namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, - python_requires=">=3.6", + python_requires=">=3.7", include_package_data=True, zip_safe=False, ) diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt deleted file mode 100644 index c3e6ad74..00000000 --- a/testing/constraints-3.6.txt +++ /dev/null @@ -1,14 +0,0 @@ -# This constraints file is used to check that lower bounds -# are correct in setup.py -# List *all* library dependencies and extras in this file. -# Pin the version to the lower bound. -# -# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", -# Then this file should have foo==1.14.0 -googleapis-common-protos==1.56.2 -protobuf==3.15.0 -google-auth==1.25.0 -requests==2.18.0 -packaging==14.3 -grpcio==1.33.2 -grpcio-status==1.33.2 From cb0329ab323a415ff9d2b854f352c7b6b49e87e9 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 10 Aug 2022 03:52:06 +0200 Subject: [PATCH 097/112] chore(deps): update actions/setup-python action to v4 (#419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update actions/setup-python action to v4 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Anthonios Partheniou Co-authored-by: Owl Bot --- .github/workflows/mypy.yml | 2 +- .github/workflows/unittest.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index e67dcbc5..f08164f6 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.7" - name: Install nox diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 4c1e8c4e..c3027cee 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -28,7 +28,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - name: Install nox @@ -55,7 +55,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" - name: Install coverage From 7472e6d10e8e7ae858be8171b477c972773f88da Mon Sep 17 00:00:00 2001 From: sai-sunder-s <4540365+sai-sunder-s@users.noreply.github.com> Date: Tue, 9 Aug 2022 18:56:45 -0700 Subject: [PATCH 098/112] doc: Update gcloud command for ADC (#406) ADC login command for gcloud is `gcloud auth application-default auth login` Co-authored-by: Anthonios Partheniou --- docs/auth.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/auth.rst b/docs/auth.rst index a9b296d4..3dcc5fd3 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -165,7 +165,7 @@ possible to call Google Cloud APIs with a user account via getting started with the ``google-cloud-*`` library. The simplest way to use credentials from a user account is via -Application Default Credentials using ``gcloud auth login`` +Application Default Credentials using ``gcloud auth application-default login`` (as mentioned above) and :func:`google.auth.default`: .. code:: python From 0dde0f9797ccbaa8a320baa7093e0577b8220905 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 24 Aug 2022 12:14:05 -0400 Subject: [PATCH 099/112] chore: remove 'pip install' statements from python_library templates [autoapprove] (#424) Source-Link: https://github.com/googleapis/synthtool/commit/1f37ce74cbc4897f35c9ba5c40393b102da913b1 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:8e84e0e0d71a0d681668461bba02c9e1394c785f31a10ae3470660235b673086 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 +- .github/workflows/docs.yml | 4 +- .github/workflows/lint.yml | 2 +- .kokoro/publish-docs.sh | 4 +- .kokoro/release.sh | 5 +- .kokoro/requirements.in | 8 + .kokoro/requirements.txt | 464 +++++++++++++++++++++++++++++++++++++ 7 files changed, 479 insertions(+), 12 deletions(-) create mode 100644 .kokoro/requirements.in create mode 100644 .kokoro/requirements.txt diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 1ce60852..1c14d7f6 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:e7bb19d47c13839fe8c147e50e02e8b6cf5da8edd1af8b82208cd6f66cc2829c -# created: 2022-07-05T18:31:20.838186805Z + digest: sha256:8e84e0e0d71a0d681668461bba02c9e1394c785f31a10ae3470660235b673086 +# created: 2022-08-24T15:24:05.205983455Z diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b46d7305..7092a139 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" - name: Install nox @@ -26,7 +26,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" - name: Install nox diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2fce29f0..eae860a2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.7" - name: Install nox diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh index 8acb14e8..1c4d6237 100755 --- a/.kokoro/publish-docs.sh +++ b/.kokoro/publish-docs.sh @@ -21,14 +21,12 @@ export PYTHONUNBUFFERED=1 export PATH="${HOME}/.local/bin:${PATH}" # Install nox -python3 -m pip install --user --upgrade --quiet nox +python3 -m pip install --require-hashes -r .kokoro/requirements.txt python3 -m nox --version # build docs nox -s docs -python3 -m pip install --user gcp-docuploader - # create metadata python3 -m docuploader create-metadata \ --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ diff --git a/.kokoro/release.sh b/.kokoro/release.sh index a00f93ec..09d841ed 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -16,12 +16,9 @@ set -eo pipefail # Start the releasetool reporter -python3 -m pip install gcp-releasetool +python3 -m pip install --require-hashes -r .kokoro/requirements.txt python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script -# Ensure that we have the latest versions of Twine, Wheel, and Setuptools. -python3 -m pip install --upgrade twine wheel setuptools - # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 diff --git a/.kokoro/requirements.in b/.kokoro/requirements.in new file mode 100644 index 00000000..7718391a --- /dev/null +++ b/.kokoro/requirements.in @@ -0,0 +1,8 @@ +gcp-docuploader +gcp-releasetool +importlib-metadata +typing-extensions +twine +wheel +setuptools +nox \ No newline at end of file diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt new file mode 100644 index 00000000..c4b824f2 --- /dev/null +++ b/.kokoro/requirements.txt @@ -0,0 +1,464 @@ +# +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: +# +# pip-compile --allow-unsafe --generate-hashes requirements.in +# +argcomplete==2.0.0 \ + --hash=sha256:6372ad78c89d662035101418ae253668445b391755cfe94ea52f1b9d22425b20 \ + --hash=sha256:cffa11ea77999bb0dd27bb25ff6dc142a6796142f68d45b1a26b11f58724561e + # via nox +attrs==22.1.0 \ + --hash=sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6 \ + --hash=sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c + # via gcp-releasetool +bleach==5.0.1 \ + --hash=sha256:085f7f33c15bd408dd9b17a4ad77c577db66d76203e5984b1bd59baeee948b2a \ + --hash=sha256:0d03255c47eb9bd2f26aa9bb7f2107732e7e8fe195ca2f64709fcf3b0a4a085c + # via readme-renderer +cachetools==5.2.0 \ + --hash=sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757 \ + --hash=sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db + # via google-auth +certifi==2022.6.15 \ + --hash=sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d \ + --hash=sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412 + # via requests +cffi==1.15.1 \ + --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ + --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \ + --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \ + --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \ + --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \ + --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \ + --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \ + --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \ + --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \ + --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \ + --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \ + --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \ + --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \ + --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \ + --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \ + --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \ + --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \ + --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \ + --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \ + --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \ + --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \ + --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \ + --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \ + --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \ + --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \ + --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \ + --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \ + --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \ + --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \ + --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \ + --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \ + --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \ + --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \ + --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \ + --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \ + --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \ + --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \ + --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \ + --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \ + --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \ + --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \ + --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \ + --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \ + --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \ + --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \ + --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \ + --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \ + --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \ + --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \ + --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \ + --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \ + --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \ + --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \ + --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \ + --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \ + --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \ + --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \ + --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \ + --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \ + --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \ + --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \ + --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \ + --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ + --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 + # via cryptography +charset-normalizer==2.1.1 \ + --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \ + --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f + # via requests +click==8.0.4 \ + --hash=sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1 \ + --hash=sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb + # via + # gcp-docuploader + # gcp-releasetool +colorlog==6.6.0 \ + --hash=sha256:344f73204009e4c83c5b6beb00b3c45dc70fcdae3c80db919e0a4171d006fde8 \ + --hash=sha256:351c51e866c86c3217f08e4b067a7974a678be78f07f85fc2d55b8babde6d94e + # via + # gcp-docuploader + # nox +commonmark==0.9.1 \ + --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ + --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 + # via rich +cryptography==37.0.4 \ + --hash=sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59 \ + --hash=sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596 \ + --hash=sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3 \ + --hash=sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5 \ + --hash=sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab \ + --hash=sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884 \ + --hash=sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82 \ + --hash=sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b \ + --hash=sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441 \ + --hash=sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa \ + --hash=sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d \ + --hash=sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b \ + --hash=sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a \ + --hash=sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6 \ + --hash=sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157 \ + --hash=sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280 \ + --hash=sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282 \ + --hash=sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67 \ + --hash=sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8 \ + --hash=sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046 \ + --hash=sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327 \ + --hash=sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9 + # via + # gcp-releasetool + # secretstorage +distlib==0.3.5 \ + --hash=sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe \ + --hash=sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c + # via virtualenv +docutils==0.19 \ + --hash=sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6 \ + --hash=sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc + # via readme-renderer +filelock==3.8.0 \ + --hash=sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc \ + --hash=sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4 + # via virtualenv +gcp-docuploader==0.6.3 \ + --hash=sha256:ba8c9d76b3bbac54b0311c503a373b00edc2dc02d6d54ea9507045adb8e870f7 \ + --hash=sha256:c0f5aaa82ce1854a386197e4e359b120ad6d4e57ae2c812fce42219a3288026b + # via -r requirements.in +gcp-releasetool==1.8.6 \ + --hash=sha256:42e51ab8e2e789bc8e22a03c09352962cd3452951c801a2230d564816630304a \ + --hash=sha256:a3518b79d1b243c494eac392a01c7fd65187fd6d52602dcab9b529bc934d4da1 + # via -r requirements.in +google-api-core==2.8.2 \ + --hash=sha256:06f7244c640322b508b125903bb5701bebabce8832f85aba9335ec00b3d02edc \ + --hash=sha256:93c6a91ccac79079ac6bbf8b74ee75db970cc899278b97d53bc012f35908cf50 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.11.0 \ + --hash=sha256:be62acaae38d0049c21ca90f27a23847245c9f161ff54ede13af2cb6afecbac9 \ + --hash=sha256:ed65ecf9f681832298e29328e1ef0a3676e3732b2e56f41532d45f70a22de0fb + # via + # gcp-releasetool + # google-api-core + # google-cloud-core + # google-cloud-storage +google-cloud-core==2.3.2 \ + --hash=sha256:8417acf6466be2fa85123441696c4badda48db314c607cf1e5d543fa8bdc22fe \ + --hash=sha256:b9529ee7047fd8d4bf4a2182de619154240df17fbe60ead399078c1ae152af9a + # via google-cloud-storage +google-cloud-storage==2.5.0 \ + --hash=sha256:19a26c66c317ce542cea0830b7e787e8dac2588b6bfa4d3fd3b871ba16305ab0 \ + --hash=sha256:382f34b91de2212e3c2e7b40ec079d27ee2e3dbbae99b75b1bcd8c63063ce235 + # via gcp-docuploader +google-crc32c==1.3.0 \ + --hash=sha256:04e7c220798a72fd0f08242bc8d7a05986b2a08a0573396187fd32c1dcdd58b3 \ + --hash=sha256:05340b60bf05b574159e9bd940152a47d38af3fb43803ffe71f11d704b7696a6 \ + --hash=sha256:12674a4c3b56b706153a358eaa1018c4137a5a04635b92b4652440d3d7386206 \ + --hash=sha256:127f9cc3ac41b6a859bd9dc4321097b1a4f6aa7fdf71b4f9227b9e3ebffb4422 \ + --hash=sha256:13af315c3a0eec8bb8b8d80b8b128cb3fcd17d7e4edafc39647846345a3f003a \ + --hash=sha256:1926fd8de0acb9d15ee757175ce7242e235482a783cd4ec711cc999fc103c24e \ + --hash=sha256:226f2f9b8e128a6ca6a9af9b9e8384f7b53a801907425c9a292553a3a7218ce0 \ + --hash=sha256:276de6273eb074a35bc598f8efbc00c7869c5cf2e29c90748fccc8c898c244df \ + --hash=sha256:318f73f5484b5671f0c7f5f63741ab020a599504ed81d209b5c7129ee4667407 \ + --hash=sha256:3bbce1be3687bbfebe29abdb7631b83e6b25da3f4e1856a1611eb21854b689ea \ + --hash=sha256:42ae4781333e331a1743445931b08ebdad73e188fd554259e772556fc4937c48 \ + --hash=sha256:58be56ae0529c664cc04a9c76e68bb92b091e0194d6e3c50bea7e0f266f73713 \ + --hash=sha256:5da2c81575cc3ccf05d9830f9e8d3c70954819ca9a63828210498c0774fda1a3 \ + --hash=sha256:6311853aa2bba4064d0c28ca54e7b50c4d48e3de04f6770f6c60ebda1e975267 \ + --hash=sha256:650e2917660e696041ab3dcd7abac160b4121cd9a484c08406f24c5964099829 \ + --hash=sha256:6a4db36f9721fdf391646685ecffa404eb986cbe007a3289499020daf72e88a2 \ + --hash=sha256:779cbf1ce375b96111db98fca913c1f5ec11b1d870e529b1dc7354b2681a8c3a \ + --hash=sha256:7f6fe42536d9dcd3e2ffb9d3053f5d05221ae3bbcefbe472bdf2c71c793e3183 \ + --hash=sha256:891f712ce54e0d631370e1f4997b3f182f3368179198efc30d477c75d1f44942 \ + --hash=sha256:95c68a4b9b7828ba0428f8f7e3109c5d476ca44996ed9a5f8aac6269296e2d59 \ + --hash=sha256:96a8918a78d5d64e07c8ea4ed2bc44354e3f93f46a4866a40e8db934e4c0d74b \ + --hash=sha256:9c3cf890c3c0ecfe1510a452a165431b5831e24160c5fcf2071f0f85ca5a47cd \ + --hash=sha256:9f58099ad7affc0754ae42e6d87443299f15d739b0ce03c76f515153a5cda06c \ + --hash=sha256:a0b9e622c3b2b8d0ce32f77eba617ab0d6768b82836391e4f8f9e2074582bf02 \ + --hash=sha256:a7f9cbea4245ee36190f85fe1814e2d7b1e5f2186381b082f5d59f99b7f11328 \ + --hash=sha256:bab4aebd525218bab4ee615786c4581952eadc16b1ff031813a2fd51f0cc7b08 \ + --hash=sha256:c124b8c8779bf2d35d9b721e52d4adb41c9bfbde45e6a3f25f0820caa9aba73f \ + --hash=sha256:c9da0a39b53d2fab3e5467329ed50e951eb91386e9d0d5b12daf593973c3b168 \ + --hash=sha256:ca60076c388728d3b6ac3846842474f4250c91efbfe5afa872d3ffd69dd4b318 \ + --hash=sha256:cb6994fff247987c66a8a4e550ef374671c2b82e3c0d2115e689d21e511a652d \ + --hash=sha256:d1c1d6236feab51200272d79b3d3e0f12cf2cbb12b208c835b175a21efdb0a73 \ + --hash=sha256:dd7760a88a8d3d705ff562aa93f8445ead54f58fd482e4f9e2bafb7e177375d4 \ + --hash=sha256:dda4d8a3bb0b50f540f6ff4b6033f3a74e8bf0bd5320b70fab2c03e512a62812 \ + --hash=sha256:e0f1ff55dde0ebcfbef027edc21f71c205845585fffe30d4ec4979416613e9b3 \ + --hash=sha256:e7a539b9be7b9c00f11ef16b55486141bc2cdb0c54762f84e3c6fc091917436d \ + --hash=sha256:eb0b14523758e37802f27b7f8cd973f5f3d33be7613952c0df904b68c4842f0e \ + --hash=sha256:ed447680ff21c14aaceb6a9f99a5f639f583ccfe4ce1a5e1d48eb41c3d6b3217 \ + --hash=sha256:f52a4ad2568314ee713715b1e2d79ab55fab11e8b304fd1462ff5cccf4264b3e \ + --hash=sha256:fbd60c6aaa07c31d7754edbc2334aef50601b7f1ada67a96eb1eb57c7c72378f \ + --hash=sha256:fc28e0db232c62ca0c3600884933178f0825c99be4474cdd645e378a10588125 \ + --hash=sha256:fe31de3002e7b08eb20823b3735b97c86c5926dd0581c7710a680b418a8709d4 \ + --hash=sha256:fec221a051150eeddfdfcff162e6db92c65ecf46cb0f7bb1bf812a1520ec026b \ + --hash=sha256:ff71073ebf0e42258a42a0b34f2c09ec384977e7f6808999102eedd5b49920e3 + # via google-resumable-media +google-resumable-media==2.3.3 \ + --hash=sha256:27c52620bd364d1c8116eaac4ea2afcbfb81ae9139fb3199652fcac1724bfb6c \ + --hash=sha256:5b52774ea7a829a8cdaa8bd2d4c3d4bc660c91b30857ab2668d0eb830f4ea8c5 + # via google-cloud-storage +googleapis-common-protos==1.56.4 \ + --hash=sha256:8eb2cbc91b69feaf23e32452a7ae60e791e09967d81d4fcc7fc388182d1bd394 \ + --hash=sha256:c25873c47279387cfdcbdafa36149887901d36202cb645a0e4f29686bf6e4417 + # via google-api-core +idna==3.3 \ + --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ + --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d + # via requests +importlib-metadata==4.12.0 \ + --hash=sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670 \ + --hash=sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23 + # via + # -r requirements.in + # twine +jeepney==0.8.0 \ + --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ + --hash=sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755 + # via + # keyring + # secretstorage +jinja2==3.1.2 \ + --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ + --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 + # via gcp-releasetool +keyring==23.8.2 \ + --hash=sha256:0d9973f8891850f1ade5f26aafd06bb16865fbbae3fc56b0defb6a14a2624003 \ + --hash=sha256:10d2a8639663fe2090705a00b8c47c687cacdf97598ea9c11456679fa974473a + # via + # gcp-releasetool + # twine +markupsafe==2.1.1 \ + --hash=sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003 \ + --hash=sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88 \ + --hash=sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5 \ + --hash=sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7 \ + --hash=sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a \ + --hash=sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603 \ + --hash=sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1 \ + --hash=sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135 \ + --hash=sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247 \ + --hash=sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6 \ + --hash=sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601 \ + --hash=sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77 \ + --hash=sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02 \ + --hash=sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e \ + --hash=sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63 \ + --hash=sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f \ + --hash=sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980 \ + --hash=sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b \ + --hash=sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812 \ + --hash=sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff \ + --hash=sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96 \ + --hash=sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1 \ + --hash=sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925 \ + --hash=sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a \ + --hash=sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6 \ + --hash=sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e \ + --hash=sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f \ + --hash=sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4 \ + --hash=sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f \ + --hash=sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3 \ + --hash=sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c \ + --hash=sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a \ + --hash=sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417 \ + --hash=sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a \ + --hash=sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a \ + --hash=sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37 \ + --hash=sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452 \ + --hash=sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933 \ + --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ + --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 + # via jinja2 +nox==2022.8.7 \ + --hash=sha256:1b894940551dc5c389f9271d197ca5d655d40bdc6ccf93ed6880e4042760a34b \ + --hash=sha256:96cca88779e08282a699d672258ec01eb7c792d35bbbf538c723172bce23212c + # via -r requirements.in +packaging==21.3 \ + --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ + --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 + # via + # gcp-releasetool + # nox +pkginfo==1.8.3 \ + --hash=sha256:848865108ec99d4901b2f7e84058b6e7660aae8ae10164e015a6dcf5b242a594 \ + --hash=sha256:a84da4318dd86f870a9447a8c98340aa06216bfc6f2b7bdc4b8766984ae1867c + # via twine +platformdirs==2.5.2 \ + --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \ + --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19 + # via virtualenv +protobuf==3.20.1 \ + --hash=sha256:06059eb6953ff01e56a25cd02cca1a9649a75a7e65397b5b9b4e929ed71d10cf \ + --hash=sha256:097c5d8a9808302fb0da7e20edf0b8d4703274d140fd25c5edabddcde43e081f \ + --hash=sha256:284f86a6207c897542d7e956eb243a36bb8f9564c1742b253462386e96c6b78f \ + --hash=sha256:32ca378605b41fd180dfe4e14d3226386d8d1b002ab31c969c366549e66a2bb7 \ + --hash=sha256:3cc797c9d15d7689ed507b165cd05913acb992d78b379f6014e013f9ecb20996 \ + --hash=sha256:62f1b5c4cd6c5402b4e2d63804ba49a327e0c386c99b1675c8a0fefda23b2067 \ + --hash=sha256:69ccfdf3657ba59569c64295b7d51325f91af586f8d5793b734260dfe2e94e2c \ + --hash=sha256:6f50601512a3d23625d8a85b1638d914a0970f17920ff39cec63aaef80a93fb7 \ + --hash=sha256:7403941f6d0992d40161aa8bb23e12575637008a5a02283a930addc0508982f9 \ + --hash=sha256:755f3aee41354ae395e104d62119cb223339a8f3276a0cd009ffabfcdd46bb0c \ + --hash=sha256:77053d28427a29987ca9caf7b72ccafee011257561259faba8dd308fda9a8739 \ + --hash=sha256:7e371f10abe57cee5021797126c93479f59fccc9693dafd6bd5633ab67808a91 \ + --hash=sha256:9016d01c91e8e625141d24ec1b20fed584703e527d28512aa8c8707f105a683c \ + --hash=sha256:9be73ad47579abc26c12024239d3540e6b765182a91dbc88e23658ab71767153 \ + --hash=sha256:adc31566d027f45efe3f44eeb5b1f329da43891634d61c75a5944e9be6dd42c9 \ + --hash=sha256:adfc6cf69c7f8c50fd24c793964eef18f0ac321315439d94945820612849c388 \ + --hash=sha256:af0ebadc74e281a517141daad9d0f2c5d93ab78e9d455113719a45a49da9db4e \ + --hash=sha256:cb29edb9eab15742d791e1025dd7b6a8f6fcb53802ad2f6e3adcb102051063ab \ + --hash=sha256:cd68be2559e2a3b84f517fb029ee611546f7812b1fdd0aa2ecc9bc6ec0e4fdde \ + --hash=sha256:cdee09140e1cd184ba9324ec1df410e7147242b94b5f8b0c64fc89e38a8ba531 \ + --hash=sha256:db977c4ca738dd9ce508557d4fce0f5aebd105e158c725beec86feb1f6bc20d8 \ + --hash=sha256:dd5789b2948ca702c17027c84c2accb552fc30f4622a98ab5c51fcfe8c50d3e7 \ + --hash=sha256:e250a42f15bf9d5b09fe1b293bdba2801cd520a9f5ea2d7fb7536d4441811d20 \ + --hash=sha256:ff8d8fa42675249bb456f5db06c00de6c2f4c27a065955917b28c4f15978b9c3 + # via + # gcp-docuploader + # gcp-releasetool + # google-api-core +py==1.11.0 \ + --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \ + --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 + # via nox +pyasn1==0.4.8 \ + --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ + --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.2.8 \ + --hash=sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e \ + --hash=sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74 + # via google-auth +pycparser==2.21 \ + --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ + --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 + # via cffi +pygments==2.13.0 \ + --hash=sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1 \ + --hash=sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42 + # via + # readme-renderer + # rich +pyjwt==2.4.0 \ + --hash=sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf \ + --hash=sha256:d42908208c699b3b973cbeb01a969ba6a96c821eefb1c5bfe4c390c01d67abba + # via gcp-releasetool +pyparsing==3.0.9 \ + --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ + --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc + # via packaging +pyperclip==1.8.2 \ + --hash=sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57 + # via gcp-releasetool +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via gcp-releasetool +readme-renderer==37.0 \ + --hash=sha256:07b7ea234e03e58f77cc222e206e6abb8f4c0435becce5104794ee591f9301c5 \ + --hash=sha256:9fa416704703e509eeb900696751c908ddeb2011319d93700d8f18baff887a69 + # via twine +requests==2.28.1 \ + --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ + --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349 + # via + # gcp-releasetool + # google-api-core + # google-cloud-storage + # requests-toolbelt + # twine +requests-toolbelt==0.9.1 \ + --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ + --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 + # via twine +rfc3986==2.0.0 \ + --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ + --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c + # via twine +rich==12.5.1 \ + --hash=sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb \ + --hash=sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca + # via twine +rsa==4.9 \ + --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ + --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21 + # via google-auth +secretstorage==3.3.3 \ + --hash=sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77 \ + --hash=sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99 + # via keyring +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via + # bleach + # gcp-docuploader + # google-auth + # python-dateutil +twine==4.0.1 \ + --hash=sha256:42026c18e394eac3e06693ee52010baa5313e4811d5a11050e7d48436cf41b9e \ + --hash=sha256:96b1cf12f7ae611a4a40b6ae8e9570215daff0611828f5fe1f37a16255ab24a0 + # via -r requirements.in +typing-extensions==4.3.0 \ + --hash=sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02 \ + --hash=sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6 + # via -r requirements.in +urllib3==1.26.12 \ + --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \ + --hash=sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997 + # via + # requests + # twine +virtualenv==20.16.3 \ + --hash=sha256:4193b7bc8a6cd23e4eb251ac64f29b4398ab2c233531e66e40b19a6b7b0d30c1 \ + --hash=sha256:d86ea0bb50e06252d79e6c241507cb904fcd66090c3271381372d6221a3970f9 + # via nox +webencodings==0.5.1 \ + --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ + --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 + # via bleach +wheel==0.37.1 \ + --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \ + --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 + # via -r requirements.in +zipp==3.8.1 \ + --hash=sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2 \ + --hash=sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +setuptools==65.2.0 \ + --hash=sha256:7f4bc85450898a09f76ebf28b72fa25bc7111f6c7d665d514a60bba9c75ef2a9 \ + --hash=sha256:a3ca5857c89f82f5c9410e8508cb32f4872a3bafd4aa7ae122a24ca33bccc750 + # via -r requirements.in From d1026e7912732384caf2c2c04da45e2ce6a8af7f Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 24 Aug 2022 19:33:52 -0400 Subject: [PATCH 100/112] chore(python): exclude path in renovate.json [autoapprove] (#425) Source-Link: https://github.com/googleapis/synthtool/commit/69fabaee9eca28af7ecaa02c86895e606fbbebd6 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:562802bfac02e012a6ac34eda282f81d06e77326b82a32d7bbb1369ff552b387 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- renovate.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 1c14d7f6..c6acdf3f 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:8e84e0e0d71a0d681668461bba02c9e1394c785f31a10ae3470660235b673086 -# created: 2022-08-24T15:24:05.205983455Z + digest: sha256:562802bfac02e012a6ac34eda282f81d06e77326b82a32d7bbb1369ff552b387 +# created: 2022-08-24T17:07:22.006876712Z diff --git a/renovate.json b/renovate.json index c21036d3..566a70f3 100644 --- a/renovate.json +++ b/renovate.json @@ -5,7 +5,7 @@ ":preserveSemverRanges", ":disableDependencyDashboard" ], - "ignorePaths": [".pre-commit-config.yaml"], + "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt"], "pip_requirements": { "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] } From 7371d025d2cdcdabb2ff670706868b34a13c4990 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 30 Aug 2022 08:52:17 -0400 Subject: [PATCH 101/112] chore(python): update dependency distlib (#429) Source-Link: https://github.com/googleapis/synthtool/commit/c4dd5953003d13b239f872d329c3146586bb417e Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ce3c1686bc81145c81dd269bd12c4025c6b275b22d14641358827334fddb1d72 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- .kokoro/requirements.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index c6acdf3f..23e106b6 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:562802bfac02e012a6ac34eda282f81d06e77326b82a32d7bbb1369ff552b387 -# created: 2022-08-24T17:07:22.006876712Z + digest: sha256:ce3c1686bc81145c81dd269bd12c4025c6b275b22d14641358827334fddb1d72 +# created: 2022-08-29T17:28:30.441852797Z diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index c4b824f2..4b29ef24 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -136,9 +136,9 @@ cryptography==37.0.4 \ # via # gcp-releasetool # secretstorage -distlib==0.3.5 \ - --hash=sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe \ - --hash=sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c +distlib==0.3.6 \ + --hash=sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46 \ + --hash=sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e # via virtualenv docutils==0.19 \ --hash=sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6 \ From c3ad8ea67447e3d8a1154d7a9221e116f60d425a Mon Sep 17 00:00:00 2001 From: Vadym Matsishevskyi <25311427+vam-google@users.noreply.github.com> Date: Tue, 30 Aug 2022 11:02:04 -0700 Subject: [PATCH 102/112] feat: Make grpc transcode logic work in terms of protobuf python objects (#428) * feat: Make grpc transcode logic work in terms of protobuf python objects (for context: [gRPC Transcoding](https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44)) Previously it worked on dictionaries only, but that causes problems. In GAPIC the dictionaries are created through the same logic as JSON (there is no better built-in way), thus applying [protobuf json mapping](https://developers.google.com/protocol-buffers/docs/proto3#json) conversion logic in the process. Unfortunately converting a protobuf object to a dictionary and to JSON, although similar, are not the same thing. Specifically the `Timestamp`, `Duration`, `FieldMask`, `uint64`, `int64`, and `*Value` protobuf messages are converted to strings for JSON (instead of being properly converted to dicts for most of those types, and `int64/uint64` converted to `int` respectively). As a result a rountrip that GAPIC was relying on (protobuf object -> dict -> transcode -> protobuf object) did not work properly. For example, when converted to dictionary, every int64 field would be converted to `string` (because it is what proto-JSON mapping spec requires), but later, when we need to rebuild a message from a transcoded dictionary that would fail with the following error: ``` TypeError: '0' has type str, but expected one of: int ``` Note, `*Rules` thing from proto-plus does not help, becuase the type may happen inside common native protobuf stub messsages (like `google.type.Money`), fields of which are outside of scope of the proto-plus custom conversion logic. Also, this change greatly simplifies the procedure of transcodding, eliminating multiple conversion steps (to and from dictionaries multiple times) making the whole logic significanly more efficient (python gapics are nutoriously known to be slow due to proto-plus stuff, so efficiency is important) and robust (JSON conversion logic does not interfere anymore with pure protobuf objects grpc transcoding) * reformat code using black * reformat code according to flake8 Co-authored-by: Anthonios Partheniou --- google/api_core/path_template.py | 60 ++++-- tests/unit/test_path_template.py | 308 ++++++++++++++++++++++++++++--- 2 files changed, 325 insertions(+), 43 deletions(-) diff --git a/google/api_core/path_template.py b/google/api_core/path_template.py index a99a4c81..2639459a 100644 --- a/google/api_core/path_template.py +++ b/google/api_core/path_template.py @@ -176,7 +176,7 @@ def get_field(request, field): """Get the value of a field from a given dictionary. Args: - request (dict): A dictionary object. + request (dict | Message): A dictionary or a Message object. field (str): The key to the request in dot notation. Returns: @@ -184,10 +184,12 @@ def get_field(request, field): """ parts = field.split(".") value = request + for part in parts: if not isinstance(value, dict): - return - value = value.get(part) + value = getattr(value, part, None) + else: + value = value.get(part) if isinstance(value, dict): return return value @@ -197,19 +199,27 @@ def delete_field(request, field): """Delete the value of a field from a given dictionary. Args: - request (dict): A dictionary object. + request (dict | Message): A dictionary object or a Message. field (str): The key to the request in dot notation. """ parts = deque(field.split(".")) while len(parts) > 1: - if not isinstance(request, dict): - return part = parts.popleft() - request = request.get(part) + if not isinstance(request, dict): + if hasattr(request, part): + request = getattr(request, part, None) + else: + return + else: + request = request.get(part) part = parts.popleft() if not isinstance(request, dict): - return - request.pop(part, None) + if hasattr(request, part): + request.ClearField(part) + else: + return + else: + request.pop(part, None) def validate(tmpl, path): @@ -237,7 +247,7 @@ def validate(tmpl, path): return True if re.match(pattern, path) is not None else False -def transcode(http_options, **request_kwargs): +def transcode(http_options, message=None, **request_kwargs): """Transcodes a grpc request pattern into a proper HTTP request following the rules outlined here, https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L44-L312 @@ -248,18 +258,20 @@ def transcode(http_options, **request_kwargs): 'body' (str): The body field name (optional) (This is a simplified representation of the proto option `google.api.http`) + message (Message) : A request object (optional) request_kwargs (dict) : A dict representing the request object Returns: dict: The transcoded request with these keys, 'method' (str) : The http method 'uri' (str) : The expanded uri - 'body' (dict) : A dict representing the body (optional) - 'query_params' (dict) : A dict mapping query parameter variables and values + 'body' (dict | Message) : A dict or a Message representing the body (optional) + 'query_params' (dict | Message) : A dict or Message mapping query parameter variables and values Raises: ValueError: If the request does not match the given template. """ + transcoded_value = message or request_kwargs for http_option in http_options: request = {} @@ -268,27 +280,35 @@ def transcode(http_options, **request_kwargs): path_fields = [ match.group("name") for match in _VARIABLE_RE.finditer(uri_template) ] - path_args = {field: get_field(request_kwargs, field) for field in path_fields} + path_args = {field: get_field(transcoded_value, field) for field in path_fields} request["uri"] = expand(uri_template, **path_args) - # Remove fields used in uri path from request - leftovers = copy.deepcopy(request_kwargs) - for path_field in path_fields: - delete_field(leftovers, path_field) if not validate(uri_template, request["uri"]) or not all(path_args.values()): continue + # Remove fields used in uri path from request + leftovers = copy.deepcopy(transcoded_value) + for path_field in path_fields: + delete_field(leftovers, path_field) + # Assign body and query params body = http_option.get("body") if body: if body == "*": request["body"] = leftovers - request["query_params"] = {} + if message: + request["query_params"] = message.__class__() + else: + request["query_params"] = {} else: try: - request["body"] = leftovers.pop(body) - except KeyError: + if message: + request["body"] = getattr(leftovers, body) + delete_field(leftovers, body) + else: + request["body"] = leftovers.pop(body) + except (KeyError, AttributeError): continue request["query_params"] = leftovers else: diff --git a/tests/unit/test_path_template.py b/tests/unit/test_path_template.py index c12b35fc..73d351c0 100644 --- a/tests/unit/test_path_template.py +++ b/tests/unit/test_path_template.py @@ -17,6 +17,7 @@ import mock import pytest +from google.api import auth_pb2 from google.api_core import path_template @@ -171,113 +172,264 @@ def test__replace_variable_with_pattern(): @pytest.mark.parametrize( - "http_options, request_kwargs, expected_result", + "http_options, message, request_kwargs, expected_result", [ [ [["get", "/v1/no/template", ""]], + None, {"foo": "bar"}, ["get", "/v1/no/template", {}, {"foo": "bar"}], ], + [ + [["get", "/v1/no/template", ""]], + auth_pb2.AuthenticationRule(selector="bar"), + {}, + [ + "get", + "/v1/no/template", + None, + auth_pb2.AuthenticationRule(selector="bar"), + ], + ], # Single templates [ [["get", "/v1/{field}", ""]], + None, {"field": "parent"}, ["get", "/v1/parent", {}, {}], ], + [ + [["get", "/v1/{selector}", ""]], + auth_pb2.AuthenticationRule(selector="parent"), + {}, + ["get", "/v1/parent", None, auth_pb2.AuthenticationRule()], + ], [ [["get", "/v1/{field.sub}", ""]], + None, {"field": {"sub": "parent"}, "foo": "bar"}, ["get", "/v1/parent", {}, {"field": {}, "foo": "bar"}], ], + [ + [["get", "/v1/{oauth.canonical_scopes}", ""]], + auth_pb2.AuthenticationRule( + selector="bar", + oauth=auth_pb2.OAuthRequirements(canonical_scopes="parent"), + ), + {}, + [ + "get", + "/v1/parent", + None, + auth_pb2.AuthenticationRule( + selector="bar", oauth=auth_pb2.OAuthRequirements() + ), + ], + ], ], ) -def test_transcode_base_case(http_options, request_kwargs, expected_result): +def test_transcode_base_case(http_options, message, request_kwargs, expected_result): http_options, expected_result = helper_test_transcode(http_options, expected_result) - result = path_template.transcode(http_options, **request_kwargs) + result = path_template.transcode(http_options, message, **request_kwargs) assert result == expected_result @pytest.mark.parametrize( - "http_options, request_kwargs, expected_result", + "http_options, message, request_kwargs, expected_result", [ [ [["get", "/v1/{field.subfield}", ""]], + None, {"field": {"subfield": "parent"}, "foo": "bar"}, ["get", "/v1/parent", {}, {"field": {}, "foo": "bar"}], ], + [ + [["get", "/v1/{oauth.canonical_scopes}", ""]], + auth_pb2.AuthenticationRule( + selector="bar", + oauth=auth_pb2.OAuthRequirements(canonical_scopes="parent"), + ), + {}, + [ + "get", + "/v1/parent", + None, + auth_pb2.AuthenticationRule( + selector="bar", oauth=auth_pb2.OAuthRequirements() + ), + ], + ], [ [["get", "/v1/{field.subfield.subsubfield}", ""]], + None, {"field": {"subfield": {"subsubfield": "parent"}}, "foo": "bar"}, ["get", "/v1/parent", {}, {"field": {"subfield": {}}, "foo": "bar"}], ], [ [["get", "/v1/{field.subfield1}/{field.subfield2}", ""]], + None, {"field": {"subfield1": "parent", "subfield2": "child"}, "foo": "bar"}, ["get", "/v1/parent/child", {}, {"field": {}, "foo": "bar"}], ], + [ + [["get", "/v1/{selector}/{oauth.canonical_scopes}", ""]], + auth_pb2.AuthenticationRule( + selector="parent", + oauth=auth_pb2.OAuthRequirements(canonical_scopes="child"), + ), + {"field": {"subfield1": "parent", "subfield2": "child"}, "foo": "bar"}, + [ + "get", + "/v1/parent/child", + None, + auth_pb2.AuthenticationRule(oauth=auth_pb2.OAuthRequirements()), + ], + ], ], ) -def test_transcode_subfields(http_options, request_kwargs, expected_result): +def test_transcode_subfields(http_options, message, request_kwargs, expected_result): http_options, expected_result = helper_test_transcode(http_options, expected_result) - result = path_template.transcode(http_options, **request_kwargs) + result = path_template.transcode(http_options, message, **request_kwargs) assert result == expected_result @pytest.mark.parametrize( - "http_options, request_kwargs, expected_result", + "http_options, message, request_kwargs, expected_result", [ # Single segment wildcard [ [["get", "/v1/{field=*}", ""]], + None, {"field": "parent"}, ["get", "/v1/parent", {}, {}], ], + [ + [["get", "/v1/{selector=*}", ""]], + auth_pb2.AuthenticationRule(selector="parent"), + {}, + ["get", "/v1/parent", None, auth_pb2.AuthenticationRule()], + ], [ [["get", "/v1/{field=a/*/b/*}", ""]], + None, {"field": "a/parent/b/child", "foo": "bar"}, ["get", "/v1/a/parent/b/child", {}, {"foo": "bar"}], ], + [ + [["get", "/v1/{selector=a/*/b/*}", ""]], + auth_pb2.AuthenticationRule( + selector="a/parent/b/child", allow_without_credential=True + ), + {}, + [ + "get", + "/v1/a/parent/b/child", + None, + auth_pb2.AuthenticationRule(allow_without_credential=True), + ], + ], # Double segment wildcard [ [["get", "/v1/{field=**}", ""]], + None, {"field": "parent/p1"}, ["get", "/v1/parent/p1", {}, {}], ], + [ + [["get", "/v1/{selector=**}", ""]], + auth_pb2.AuthenticationRule(selector="parent/p1"), + {}, + ["get", "/v1/parent/p1", None, auth_pb2.AuthenticationRule()], + ], [ [["get", "/v1/{field=a/**/b/**}", ""]], + None, {"field": "a/parent/p1/b/child/c1", "foo": "bar"}, ["get", "/v1/a/parent/p1/b/child/c1", {}, {"foo": "bar"}], ], + [ + [["get", "/v1/{selector=a/**/b/**}", ""]], + auth_pb2.AuthenticationRule( + selector="a/parent/p1/b/child/c1", allow_without_credential=True + ), + {}, + [ + "get", + "/v1/a/parent/p1/b/child/c1", + None, + auth_pb2.AuthenticationRule(allow_without_credential=True), + ], + ], # Combined single and double segment wildcard [ [["get", "/v1/{field=a/*/b/**}", ""]], + None, {"field": "a/parent/b/child/c1"}, ["get", "/v1/a/parent/b/child/c1", {}, {}], ], + [ + [["get", "/v1/{selector=a/*/b/**}", ""]], + auth_pb2.AuthenticationRule(selector="a/parent/b/child/c1"), + {}, + ["get", "/v1/a/parent/b/child/c1", None, auth_pb2.AuthenticationRule()], + ], [ [["get", "/v1/{field=a/**/b/*}/v2/{name}", ""]], + None, {"field": "a/parent/p1/b/child", "name": "first", "foo": "bar"}, ["get", "/v1/a/parent/p1/b/child/v2/first", {}, {"foo": "bar"}], ], + [ + [["get", "/v1/{selector=a/**/b/*}/v2/{oauth.canonical_scopes}", ""]], + auth_pb2.AuthenticationRule( + selector="a/parent/p1/b/child", + oauth=auth_pb2.OAuthRequirements(canonical_scopes="first"), + ), + {"field": "a/parent/p1/b/child", "name": "first", "foo": "bar"}, + [ + "get", + "/v1/a/parent/p1/b/child/v2/first", + None, + auth_pb2.AuthenticationRule(oauth=auth_pb2.OAuthRequirements()), + ], + ], ], ) -def test_transcode_with_wildcard(http_options, request_kwargs, expected_result): +def test_transcode_with_wildcard( + http_options, message, request_kwargs, expected_result +): http_options, expected_result = helper_test_transcode(http_options, expected_result) - result = path_template.transcode(http_options, **request_kwargs) + result = path_template.transcode(http_options, message, **request_kwargs) assert result == expected_result @pytest.mark.parametrize( - "http_options, request_kwargs, expected_result", + "http_options, message, request_kwargs, expected_result", [ # Single field body [ [["post", "/v1/no/template", "data"]], + None, {"data": {"id": 1, "info": "some info"}, "foo": "bar"}, ["post", "/v1/no/template", {"id": 1, "info": "some info"}, {"foo": "bar"}], ], + [ + [["post", "/v1/no/template", "oauth"]], + auth_pb2.AuthenticationRule( + selector="bar", + oauth=auth_pb2.OAuthRequirements(canonical_scopes="child"), + ), + {}, + [ + "post", + "/v1/no/template", + auth_pb2.OAuthRequirements(canonical_scopes="child"), + auth_pb2.AuthenticationRule(selector="bar"), + ], + ], [ [["post", "/v1/{field=a/*}/b/{name=**}", "data"]], + None, { "field": "a/parent", "name": "first/last", @@ -291,9 +443,29 @@ def test_transcode_with_wildcard(http_options, request_kwargs, expected_result): {"foo": "bar"}, ], ], + [ + [["post", "/v1/{selector=a/*}/b/{oauth.canonical_scopes=**}", "oauth"]], + auth_pb2.AuthenticationRule( + selector="a/parent", + allow_without_credential=True, + requirements=[auth_pb2.AuthRequirement(provider_id="p")], + oauth=auth_pb2.OAuthRequirements(canonical_scopes="first/last"), + ), + {}, + [ + "post", + "/v1/a/parent/b/first/last", + auth_pb2.OAuthRequirements(), + auth_pb2.AuthenticationRule( + requirements=[auth_pb2.AuthRequirement(provider_id="p")], + allow_without_credential=True, + ), + ], + ], # Wildcard body [ [["post", "/v1/{field=a/*}/b/{name=**}", "*"]], + None, { "field": "a/parent", "name": "first/last", @@ -307,16 +479,38 @@ def test_transcode_with_wildcard(http_options, request_kwargs, expected_result): {}, ], ], + [ + [["post", "/v1/{selector=a/*}/b/{oauth.canonical_scopes=**}", "*"]], + auth_pb2.AuthenticationRule( + selector="a/parent", + allow_without_credential=True, + oauth=auth_pb2.OAuthRequirements(canonical_scopes="first/last"), + ), + { + "field": "a/parent", + "name": "first/last", + "data": {"id": 1, "info": "some info"}, + "foo": "bar", + }, + [ + "post", + "/v1/a/parent/b/first/last", + auth_pb2.AuthenticationRule( + allow_without_credential=True, oauth=auth_pb2.OAuthRequirements() + ), + auth_pb2.AuthenticationRule(), + ], + ], ], ) -def test_transcode_with_body(http_options, request_kwargs, expected_result): +def test_transcode_with_body(http_options, message, request_kwargs, expected_result): http_options, expected_result = helper_test_transcode(http_options, expected_result) - result = path_template.transcode(http_options, **request_kwargs) + result = path_template.transcode(http_options, message, **request_kwargs) assert result == expected_result @pytest.mark.parametrize( - "http_options, request_kwargs, expected_result", + "http_options, message, request_kwargs, expected_result", [ # Additional bindings [ @@ -324,6 +518,7 @@ def test_transcode_with_body(http_options, request_kwargs, expected_result): ["post", "/v1/{field=a/*}/b/{name=**}", "extra_data"], ["post", "/v1/{field=a/*}/b/{name=**}", "*"], ], + None, { "field": "a/parent", "name": "first/last", @@ -337,38 +532,105 @@ def test_transcode_with_body(http_options, request_kwargs, expected_result): {}, ], ], + [ + [ + [ + "post", + "/v1/{selector=a/*}/b/{oauth.canonical_scopes=**}", + "extra_data", + ], + ["post", "/v1/{selector=a/*}/b/{oauth.canonical_scopes=**}", "*"], + ], + auth_pb2.AuthenticationRule( + selector="a/parent", + allow_without_credential=True, + oauth=auth_pb2.OAuthRequirements(canonical_scopes="first/last"), + ), + {}, + [ + "post", + "/v1/a/parent/b/first/last", + auth_pb2.AuthenticationRule( + allow_without_credential=True, oauth=auth_pb2.OAuthRequirements() + ), + auth_pb2.AuthenticationRule(), + ], + ], [ [ ["get", "/v1/{field=a/*}/b/{name=**}", ""], ["get", "/v1/{field=a/*}/b/first/last", ""], ], + None, {"field": "a/parent", "foo": "bar"}, ["get", "/v1/a/parent/b/first/last", {}, {"foo": "bar"}], ], + [ + [ + ["get", "/v1/{selector=a/*}/b/{oauth.allow_without_credential=**}", ""], + ["get", "/v1/{selector=a/*}/b/first/last", ""], + ], + auth_pb2.AuthenticationRule( + selector="a/parent", + allow_without_credential=True, + oauth=auth_pb2.OAuthRequirements(), + ), + {}, + [ + "get", + "/v1/a/parent/b/first/last", + None, + auth_pb2.AuthenticationRule( + allow_without_credential=True, oauth=auth_pb2.OAuthRequirements() + ), + ], + ], ], ) def test_transcode_with_additional_bindings( - http_options, request_kwargs, expected_result + http_options, message, request_kwargs, expected_result ): http_options, expected_result = helper_test_transcode(http_options, expected_result) - result = path_template.transcode(http_options, **request_kwargs) + result = path_template.transcode(http_options, message, **request_kwargs) assert result == expected_result @pytest.mark.parametrize( - "http_options, request_kwargs", + "http_options, message, request_kwargs", [ - [[["get", "/v1/{name}", ""]], {"foo": "bar"}], - [[["get", "/v1/{name}", ""]], {"name": "first/last"}], - [[["get", "/v1/{name=mr/*/*}", ""]], {"name": "first/last"}], - [[["post", "/v1/{name}", "data"]], {"name": "first/last"}], - [[["post", "/v1/{first_name}", "data"]], {"last_name": "last"}], + [[["get", "/v1/{name}", ""]], None, {"foo": "bar"}], + [[["get", "/v1/{selector}", ""]], auth_pb2.AuthenticationRule(), {}], + [[["get", "/v1/{name}", ""]], auth_pb2.AuthenticationRule(), {}], + [[["get", "/v1/{name}", ""]], None, {"name": "first/last"}], + [ + [["get", "/v1/{selector}", ""]], + auth_pb2.AuthenticationRule(selector="first/last"), + {}, + ], + [[["get", "/v1/{name=mr/*/*}", ""]], None, {"name": "first/last"}], + [ + [["get", "/v1/{selector=mr/*/*}", ""]], + auth_pb2.AuthenticationRule(selector="first/last"), + {}, + ], + [[["post", "/v1/{name}", "data"]], None, {"name": "first/last"}], + [ + [["post", "/v1/{selector}", "data"]], + auth_pb2.AuthenticationRule(selector="first"), + {}, + ], + [[["post", "/v1/{first_name}", "data"]], None, {"last_name": "last"}], + [ + [["post", "/v1/{first_name}", ""]], + auth_pb2.AuthenticationRule(selector="first"), + {}, + ], ], ) -def test_transcode_fails(http_options, request_kwargs): +def test_transcode_fails(http_options, message, request_kwargs): http_options, _ = helper_test_transcode(http_options, range(4)) with pytest.raises(ValueError): - path_template.transcode(http_options, **request_kwargs) + path_template.transcode(http_options, message, **request_kwargs) def helper_test_transcode(http_options_list, expected_result_list): From 29aa9efc8c6e7193bdbf00f0e6d8cb5840ad6b6e Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 1 Sep 2022 18:28:12 +0000 Subject: [PATCH 103/112] chore: fix path to requirements.txt in release script [autoapprove] (#430) Source-Link: https://github.com/googleapis/synthtool/commit/fdba3ed145bdb2f4f3eff434d4284b1d03b80d34 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:1f0dbd02745fb7cf255563dab5968345989308544e52b7f460deadd5e78e63b0 --- .github/.OwlBot.lock.yaml | 3 +-- .kokoro/release.sh | 2 +- .kokoro/requirements.txt | 24 ++++++++++++------------ 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 23e106b6..0d9eb2af 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ce3c1686bc81145c81dd269bd12c4025c6b275b22d14641358827334fddb1d72 -# created: 2022-08-29T17:28:30.441852797Z + digest: sha256:1f0dbd02745fb7cf255563dab5968345989308544e52b7f460deadd5e78e63b0 diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 09d841ed..697f7e6d 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -16,7 +16,7 @@ set -eo pipefail # Start the releasetool reporter -python3 -m pip install --require-hashes -r .kokoro/requirements.txt +python3 -m pip install --require-hashes -r github/python-api-core/.kokoro/requirements.txt python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script # Disable buffering, so that the logs stream through. diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 4b29ef24..92b2f727 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -100,9 +100,9 @@ click==8.0.4 \ # via # gcp-docuploader # gcp-releasetool -colorlog==6.6.0 \ - --hash=sha256:344f73204009e4c83c5b6beb00b3c45dc70fcdae3c80db919e0a4171d006fde8 \ - --hash=sha256:351c51e866c86c3217f08e4b067a7974a678be78f07f85fc2d55b8babde6d94e +colorlog==6.7.0 \ + --hash=sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662 \ + --hash=sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5 # via # gcp-docuploader # nox @@ -152,9 +152,9 @@ gcp-docuploader==0.6.3 \ --hash=sha256:ba8c9d76b3bbac54b0311c503a373b00edc2dc02d6d54ea9507045adb8e870f7 \ --hash=sha256:c0f5aaa82ce1854a386197e4e359b120ad6d4e57ae2c812fce42219a3288026b # via -r requirements.in -gcp-releasetool==1.8.6 \ - --hash=sha256:42e51ab8e2e789bc8e22a03c09352962cd3452951c801a2230d564816630304a \ - --hash=sha256:a3518b79d1b243c494eac392a01c7fd65187fd6d52602dcab9b529bc934d4da1 +gcp-releasetool==1.8.7 \ + --hash=sha256:3d2a67c9db39322194afb3b427e9cb0476ce8f2a04033695f0aeb63979fc2b37 \ + --hash=sha256:5e4d28f66e90780d77f3ecf1e9155852b0c3b13cbccb08ab07e66b2357c8da8d # via -r requirements.in google-api-core==2.8.2 \ --hash=sha256:06f7244c640322b508b125903bb5701bebabce8832f85aba9335ec00b3d02edc \ @@ -251,9 +251,9 @@ jinja2==3.1.2 \ --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 # via gcp-releasetool -keyring==23.8.2 \ - --hash=sha256:0d9973f8891850f1ade5f26aafd06bb16865fbbae3fc56b0defb6a14a2624003 \ - --hash=sha256:10d2a8639663fe2090705a00b8c47c687cacdf97598ea9c11456679fa974473a +keyring==23.9.0 \ + --hash=sha256:4c32a31174faaee48f43a7e2c7e9c3216ec5e95acf22a2bebfb4a1d05056ee44 \ + --hash=sha256:98f060ec95ada2ab910c195a2d4317be6ef87936a766b239c46aa3c7aac4f0db # via # gcp-releasetool # twine @@ -440,9 +440,9 @@ urllib3==1.26.12 \ # via # requests # twine -virtualenv==20.16.3 \ - --hash=sha256:4193b7bc8a6cd23e4eb251ac64f29b4398ab2c233531e66e40b19a6b7b0d30c1 \ - --hash=sha256:d86ea0bb50e06252d79e6c241507cb904fcd66090c3271381372d6221a3970f9 +virtualenv==20.16.4 \ + --hash=sha256:014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782 \ + --hash=sha256:035ed57acce4ac35c82c9d8802202b0e71adac011a511ff650cbcf9635006a22 # via nox webencodings==0.5.1 \ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ From 8c19609d6244930bd91fd5f40ef9b5b65584c4a5 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 1 Sep 2022 16:22:35 -0400 Subject: [PATCH 104/112] fix: restore support for grpcio-gcp (#418) docs: add a note that grpcio-gcp is only supported in environments with protobuf < 4.x.x docs: raise DeprecationWarning when 'grpcio-gcp' is used fix(deps): require protobuf >= 3.20.1 --- .github/sync-repo-settings.yaml | 4 ++ .github/workflows/unittest.yml | 2 +- google/api_core/grpc_helpers.py | 28 ++++++++++- noxfile.py | 15 ++++++ setup.py | 4 +- testing/constraints-3.7.txt | 3 +- testing/constraints-3.8.txt | 2 + tests/unit/test_grpc_helpers.py | 88 ++++++++++++++++++++++++++++----- 8 files changed, 129 insertions(+), 17 deletions(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index e2d70f90..6d2c2a0e 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -11,6 +11,10 @@ branchProtectionRules: # No Kokoro: the following are Github actions - 'lint' - 'mypy' + - 'unit_grpc_gcp-3.7' + - 'unit_grpc_gcp-3.8' + - 'unit_grpc_gcp-3.9' + - 'unit_grpc_gcp-3.10' - 'unit-3.7' - 'unit-3.8' - 'unit-3.9' diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index c3027cee..cd1d4d60 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - option: ["", "_wo_grpc"] + option: ["", "_grpc_gcp", "_wo_grpc"] python: - "3.7" - "3.8" diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index 47d27726..bf04ae4c 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -16,6 +16,7 @@ import collections import functools +import warnings import grpc @@ -24,6 +25,27 @@ import google.auth.credentials import google.auth.transport.grpc import google.auth.transport.requests +import google.protobuf + +PROTOBUF_VERSION = google.protobuf.__version__ + +# The grpcio-gcp package only has support for protobuf < 4 +if PROTOBUF_VERSION[0:2] == "3.": + try: + import grpc_gcp + + warnings.warn( + """Support for grpcio-gcp is deprecated. This feature will be + removed from `google-api-core` after January 1, 2024. If you need to + continue to use this feature, please pin to a specific version of + `google-api-core`.""", + DeprecationWarning, + ) + HAS_GRPC_GCP = True + except ImportError: + HAS_GRPC_GCP = False +else: + HAS_GRPC_GCP = False # The list of gRPC Callable interfaces that return iterators. @@ -275,7 +297,9 @@ def create_channel( default_scopes (Sequence[str]): Default scopes passed by a Google client library. Use 'scopes' for user-defined scopes. default_host (str): The default endpoint. e.g., "pubsub.googleapis.com". - kwargs: Additional key-word args passed to :func:`grpc.secure_channel`. + kwargs: Additional key-word args passed to + :func:`grpc_gcp.secure_channel` or :func:`grpc.secure_channel`. + Note: `grpc_gcp` is only supported in environments with protobuf < 4.0.0. Returns: grpc.Channel: The created channel. @@ -294,6 +318,8 @@ def create_channel( default_host=default_host, ) + if HAS_GRPC_GCP: + return grpc_gcp.secure_channel(target, composite_credentials, **kwargs) return grpc.secure_channel(target, composite_credentials, **kwargs) diff --git a/noxfile.py b/noxfile.py index d0db9914..2d8f1e02 100644 --- a/noxfile.py +++ b/noxfile.py @@ -32,6 +32,7 @@ # 'docfx' is excluded since it only needs to run in 'docs-presubmit' nox.options.sessions = [ "unit", + "unit_grpc_gcp", "unit_wo_grpc", "cover", "pytype", @@ -142,6 +143,20 @@ def unit(session): default(session) +@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) +def unit_grpc_gcp(session): + """Run the unit test suite with grpcio-gcp installed.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + # Install grpcio-gcp + session.install("-e", ".[grpcgcp]", "-c", constraints_path) + # Install protobuf < 4.0.0 + session.install("protobuf<4.0.0") + + default(session) + + @nox.session(python=["3.8", "3.10"]) def unit_wo_grpc(session): """Run the unit test suite w/o grpcio installed""" diff --git a/setup.py b/setup.py index 4919e5f8..2dd2a0cd 100644 --- a/setup.py +++ b/setup.py @@ -30,12 +30,14 @@ release_status = "Development Status :: 5 - Production/Stable" dependencies = [ "googleapis-common-protos >= 1.56.2, < 2.0dev", - "protobuf >= 3.15.0, <5.0.0dev", + "protobuf >= 3.20.1, <5.0.0dev", "google-auth >= 1.25.0, < 3.0dev", "requests >= 2.18.0, < 3.0.0dev", ] extras = { "grpc": ["grpcio >= 1.33.2, < 2.0dev", "grpcio-status >= 1.33.2, < 2.0dev"], + "grpcgcp": "grpcio-gcp >= 0.2.2, < 1.0dev", + "grpcio-gcp": "grpcio-gcp >= 0.2.2, < 1.0dev", } diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index c3e6ad74..fe671145 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -6,9 +6,10 @@ # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 googleapis-common-protos==1.56.2 -protobuf==3.15.0 +protobuf==3.20.1 google-auth==1.25.0 requests==2.18.0 packaging==14.3 grpcio==1.33.2 grpcio-status==1.33.2 +grpcio-gcp==0.2.2 diff --git a/testing/constraints-3.8.txt b/testing/constraints-3.8.txt index e69de29b..8d760bbd 100644 --- a/testing/constraints-3.8.txt +++ b/testing/constraints-3.8.txt @@ -0,0 +1,2 @@ +googleapis-common-protos==1.56.3 +protobuf==4.21.5 \ No newline at end of file diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py index 649072f0..8b9fd9f1 100644 --- a/tests/unit/test_grpc_helpers.py +++ b/tests/unit/test_grpc_helpers.py @@ -365,7 +365,10 @@ def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_c default.assert_called_once_with(scopes=None, default_scopes=None) - grpc_secure_channel.assert_called_once_with(target, composite_creds) + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("google.auth.transport.grpc.AuthMetadataPlugin", autospec=True) @@ -397,7 +400,10 @@ def test_create_channel_implicit_with_default_host( mock.sentinel.credentials, mock.sentinel.Request, default_host=default_host ) - grpc_secure_channel.assert_called_once_with(target, composite_creds) + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -420,7 +426,11 @@ def test_create_channel_implicit_with_ssl_creds( composite_creds_call.assert_called_once_with(ssl_creds, mock.ANY) composite_creds = composite_creds_call.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -442,7 +452,10 @@ def test_create_channel_implicit_with_scopes( default.assert_called_once_with(scopes=["one", "two"], default_scopes=None) - grpc_secure_channel.assert_called_once_with(target, composite_creds) + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -464,7 +477,10 @@ def test_create_channel_implicit_with_default_scopes( default.assert_called_once_with(scopes=None, default_scopes=["three", "four"]) - grpc_secure_channel.assert_called_once_with(target, composite_creds) + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) def test_create_channel_explicit_with_duplicate_credentials(): @@ -492,7 +508,11 @@ def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_cred ) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -512,7 +532,11 @@ def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_cal credentials.with_scopes.assert_called_once_with(scopes, default_scopes=None) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -536,7 +560,11 @@ def test_create_channel_explicit_default_scopes( ) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -558,7 +586,11 @@ def test_create_channel_explicit_with_quota_project( credentials.with_quota_project.assert_called_once_with("project-foo") assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -583,7 +615,11 @@ def test_create_channel_with_credentials_file( ) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -611,7 +647,11 @@ def test_create_channel_with_credentials_file_and_scopes( ) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) @mock.patch("grpc.composite_channel_credentials") @@ -639,11 +679,33 @@ def test_create_channel_with_credentials_file_and_default_scopes( ) assert channel is grpc_secure_channel.return_value - grpc_secure_channel.assert_called_once_with(target, composite_creds) + + if grpc_helpers.HAS_GRPC_GCP: + grpc_secure_channel.assert_called_once_with(target, composite_creds, None) + else: + grpc_secure_channel.assert_called_once_with(target, composite_creds) + + +@pytest.mark.skipif( + not grpc_helpers.HAS_GRPC_GCP, reason="grpc_gcp module not available" +) +@mock.patch("grpc_gcp.secure_channel") +def test_create_channel_with_grpc_gcp(grpc_gcp_secure_channel): + target = "example.com:443" + scopes = ["test_scope"] + + credentials = mock.create_autospec(google.auth.credentials.Scoped, instance=True) + credentials.requires_scopes = True + + grpc_helpers.create_channel(target, credentials=credentials, scopes=scopes) + grpc_gcp_secure_channel.assert_called() + + credentials.with_scopes.assert_called_once_with(scopes, default_scopes=None) +@pytest.mark.skipif(grpc_helpers.HAS_GRPC_GCP, reason="grpc_gcp module not available") @mock.patch("grpc.secure_channel") -def test_create_channel(grpc_secure_channel): +def test_create_channel_without_grpc_gcp(grpc_secure_channel): target = "example.com:443" scopes = ["test_scope"] From 922f46852ca4420a4ac217db8b814561735a4415 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 1 Sep 2022 16:31:54 -0400 Subject: [PATCH 105/112] chore(main): release 2.9.0 (#422) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 13 +++++++++++++ google/api_core/version.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16e22ab8..90826869 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.9.0](https://github.com/googleapis/python-api-core/compare/v2.8.2...v2.9.0) (2022-09-01) + + +### Features + +* Make grpc transcode logic work in terms of protobuf python objects ([#428](https://github.com/googleapis/python-api-core/issues/428)) ([c3ad8ea](https://github.com/googleapis/python-api-core/commit/c3ad8ea67447e3d8a1154d7a9221e116f60d425a)) + + +### Bug Fixes + +* Require python 3.7+ ([#410](https://github.com/googleapis/python-api-core/issues/410)) ([7ddb8c0](https://github.com/googleapis/python-api-core/commit/7ddb8c00e6be7ab6905a9a802ad1c3063fbfa46c)) +* Restore support for grpcio-gcp ([#418](https://github.com/googleapis/python-api-core/issues/418)) ([8c19609](https://github.com/googleapis/python-api-core/commit/8c19609d6244930bd91fd5f40ef9b5b65584c4a5)) + ## [2.8.2](https://github.com/googleapis/python-api-core/compare/v2.8.1...v2.8.2) (2022-06-13) diff --git a/google/api_core/version.py b/google/api_core/version.py index 839a77a1..b2a8c553 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.8.2" +__version__ = "2.9.0" From 9066ed49dbdb47b2245507eb2c25acd97984be54 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 1 Sep 2022 19:27:59 -0400 Subject: [PATCH 106/112] chore(python): update .kokoro/requirements.txt (#431) Source-Link: https://github.com/googleapis/synthtool/commit/703554a14c7479542335b62fa69279f93a9e38ec Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:94961fdc5c9ca6d13530a6a414a49d2f607203168215d074cdb0a1df9ec31c0b Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/requirements.txt | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 0d9eb2af..2fa0f7c4 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:1f0dbd02745fb7cf255563dab5968345989308544e52b7f460deadd5e78e63b0 + digest: sha256:94961fdc5c9ca6d13530a6a414a49d2f607203168215d074cdb0a1df9ec31c0b diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 92b2f727..385f2d4d 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -241,6 +241,10 @@ importlib-metadata==4.12.0 \ # via # -r requirements.in # twine +jaraco-classes==3.2.2 \ + --hash=sha256:6745f113b0b588239ceb49532aa09c3ebb947433ce311ef2f8e3ad64ebb74594 \ + --hash=sha256:e6ef6fd3fcf4579a7a019d87d1e56a883f4e4c35cfe925f86731abc58804e647 + # via keyring jeepney==0.8.0 \ --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ --hash=sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755 @@ -299,6 +303,10 @@ markupsafe==2.1.1 \ --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 # via jinja2 +more-itertools==8.14.0 \ + --hash=sha256:1bc4f91ee5b1b31ac7ceacc17c09befe6a40a503907baf9c839c229b5095cfd2 \ + --hash=sha256:c09443cd3d5438b8dafccd867a6bc1cb0894389e90cb53d227456b0b0bccb750 + # via jaraco-classes nox==2022.8.7 \ --hash=sha256:1b894940551dc5c389f9271d197ca5d655d40bdc6ccf93ed6880e4042760a34b \ --hash=sha256:96cca88779e08282a699d672258ec01eb7c792d35bbbf538c723172bce23212c From 83678e94e1081f9087b19c43f26fad4774184d66 Mon Sep 17 00:00:00 2001 From: Victor Chudnovsky Date: Thu, 1 Sep 2022 17:27:04 -0700 Subject: [PATCH 107/112] feat: add 'strict' to flatten_query_params to lower-case bools (#433) * feat: add 'strict' to flatten_query_params to lower-case bools * pylint Co-authored-by: Anthonios Partheniou --- google/api_core/rest_helpers.py | 49 +++++++++++++++++++++------------ tests/unit/test_rest_helpers.py | 19 ++++++++++++- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/google/api_core/rest_helpers.py b/google/api_core/rest_helpers.py index 23fb614f..a78822f1 100644 --- a/google/api_core/rest_helpers.py +++ b/google/api_core/rest_helpers.py @@ -18,8 +18,8 @@ import operator -def flatten_query_params(obj): - """Flatten a nested dict into a list of (name,value) tuples. +def flatten_query_params(obj, strict=False): + """Flatten a dict into a list of (name,value) tuples. The result is suitable for setting query params on an http request. @@ -28,9 +28,10 @@ def flatten_query_params(obj): >>> obj = {'a': ... {'b': ... {'c': ['x', 'y', 'z']} }, - ... 'd': 'uvw', } - >>> flatten_query_params(obj) - [('a.b.c', 'x'), ('a.b.c', 'y'), ('a.b.c', 'z'), ('d', 'uvw')] + ... 'd': 'uvw', + ... 'e': True, } + >>> flatten_query_params(obj, strict=True) + [('a.b.c', 'x'), ('a.b.c', 'y'), ('a.b.c', 'z'), ('d', 'uvw'), ('e', 'true')] Note that, as described in https://github.com/googleapis/googleapis/blob/48d9fb8c8e287c472af500221c6450ecd45d7d39/google/api/http.proto#L117, @@ -38,7 +39,9 @@ def flatten_query_params(obj): This is enforced in this function. Args: - obj: a nested dictionary (from json), or None + obj: a possibly nested dictionary (from json), or None + strict: a bool, defaulting to False, to enforce that all values in the + result tuples be strings and, if boolean, lower-cased. Returns: a list of tuples, with each tuple having a (possibly) multi-part name and a scalar value. @@ -51,17 +54,17 @@ def flatten_query_params(obj): if obj is not None and not isinstance(obj, dict): raise TypeError("flatten_query_params must be called with dict object") - return _flatten(obj, key_path=[]) + return _flatten(obj, key_path=[], strict=strict) -def _flatten(obj, key_path): +def _flatten(obj, key_path, strict=False): if obj is None: return [] if isinstance(obj, dict): - return _flatten_dict(obj, key_path=key_path) + return _flatten_dict(obj, key_path=key_path, strict=strict) if isinstance(obj, list): - return _flatten_list(obj, key_path=key_path) - return _flatten_value(obj, key_path=key_path) + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) def _is_primitive_value(obj): @@ -74,21 +77,33 @@ def _is_primitive_value(obj): return True -def _flatten_value(obj, key_path): - return [(".".join(key_path), obj)] +def _flatten_value(obj, key_path, strict=False): + return [(".".join(key_path), _canonicalize(obj, strict=strict))] -def _flatten_dict(obj, key_path): - items = (_flatten(value, key_path=key_path + [key]) for key, value in obj.items()) +def _flatten_dict(obj, key_path, strict=False): + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) return functools.reduce(operator.concat, items, []) -def _flatten_list(elems, key_path): +def _flatten_list(elems, key_path, strict=False): # Only lists of scalar values are supported. # The name (key_path) is repeated for each value. items = ( - _flatten_value(elem, key_path=key_path) + _flatten_value(elem, key_path=key_path, strict=strict) for elem in elems if _is_primitive_value(elem) ) return functools.reduce(operator.concat, items, []) + + +def _canonicalize(obj, strict=False): + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj diff --git a/tests/unit/test_rest_helpers.py b/tests/unit/test_rest_helpers.py index 5932fa55..ff1a43f0 100644 --- a/tests/unit/test_rest_helpers.py +++ b/tests/unit/test_rest_helpers.py @@ -36,9 +36,26 @@ def test_flatten_empty_dict(): def test_flatten_simple_dict(): - assert rest_helpers.flatten_query_params({"a": "abc", "b": "def"}) == [ + obj = {"a": "abc", "b": "def", "c": True, "d": False, "e": 10, "f": -3.76} + assert rest_helpers.flatten_query_params(obj) == [ ("a", "abc"), ("b", "def"), + ("c", True), + ("d", False), + ("e", 10), + ("f", -3.76), + ] + + +def test_flatten_simple_dict_strict(): + obj = {"a": "abc", "b": "def", "c": True, "d": False, "e": 10, "f": -3.76} + assert rest_helpers.flatten_query_params(obj, strict=True) == [ + ("a", "abc"), + ("b", "def"), + ("c", "true"), + ("d", "false"), + ("e", "10"), + ("f", "-3.76"), ] From cea67dafecc71481b0c362763da354a541966c79 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 1 Sep 2022 17:43:34 -0700 Subject: [PATCH 108/112] chore(main): release 2.10.0 (#434) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ google/api_core/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90826869..268cb4d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.10.0](https://github.com/googleapis/python-api-core/compare/v2.9.0...v2.10.0) (2022-09-02) + + +### Features + +* Add 'strict' to flatten_query_params to lower-case bools ([#433](https://github.com/googleapis/python-api-core/issues/433)) ([83678e9](https://github.com/googleapis/python-api-core/commit/83678e94e1081f9087b19c43f26fad4774184d66)) + ## [2.9.0](https://github.com/googleapis/python-api-core/compare/v2.8.2...v2.9.0) (2022-09-01) diff --git a/google/api_core/version.py b/google/api_core/version.py index b2a8c553..13e710fc 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.9.0" +__version__ = "2.10.0" From fe617c205918a3e4dfddeb06123e70540898032e Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 18:44:27 +0000 Subject: [PATCH 109/112] chore(python): exclude setup.py in renovate config (#436) Source-Link: https://github.com/googleapis/synthtool/commit/56da63e80c384a871356d1ea6640802017f213b4 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:993a058718e84a82fda04c3177e58f0a43281a996c7c395e0a56ccc4d6d210d7 --- .github/.OwlBot.lock.yaml | 2 +- renovate.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 2fa0f7c4..b8dcb4a4 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:94961fdc5c9ca6d13530a6a414a49d2f607203168215d074cdb0a1df9ec31c0b + digest: sha256:993a058718e84a82fda04c3177e58f0a43281a996c7c395e0a56ccc4d6d210d7 diff --git a/renovate.json b/renovate.json index 566a70f3..39b2a0ec 100644 --- a/renovate.json +++ b/renovate.json @@ -5,7 +5,7 @@ ":preserveSemverRanges", ":disableDependencyDashboard" ], - "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt"], + "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py"], "pip_requirements": { "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] } From cc79857f32598a1d7fd7d9eef7ad7c930bd6d3ab Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 6 Sep 2022 14:49:41 +0000 Subject: [PATCH 110/112] enable github actions checks --- .github/workflows/docs.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/mypy.yml | 2 +- .github/workflows/unittest.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7092a139..6dd32fec 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,7 +1,7 @@ on: pull_request: branches: - - main + - v1 name: docs jobs: docs: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index eae860a2..cabd0e5b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,7 +1,7 @@ on: pull_request: branches: - - main + - v1 name: lint jobs: lint: diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index f08164f6..d9c98dba 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -1,7 +1,7 @@ on: pull_request: branches: - - main + - v1 name: mypy jobs: mypy: diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index cd1d4d60..bdce7eb7 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -3,7 +3,7 @@ name: "Unit tests" on: pull_request: branches: - - main + - v1 jobs: run-unittests: From ac301cf5e36fb818ca90b96d7c4bb3201fed4c0b Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 6 Sep 2022 14:52:52 +0000 Subject: [PATCH 111/112] remove constraints for python 3.8 --- testing/constraints-3.8.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/testing/constraints-3.8.txt b/testing/constraints-3.8.txt index 8d760bbd..e69de29b 100644 --- a/testing/constraints-3.8.txt +++ b/testing/constraints-3.8.txt @@ -1,2 +0,0 @@ -googleapis-common-protos==1.56.3 -protobuf==4.21.5 \ No newline at end of file From 238e41ba4f04f4cdf1f5adf74a7a7dd3007c921b Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 6 Sep 2022 14:56:29 +0000 Subject: [PATCH 112/112] ci: fix cover --- google/api_core/grpc_helpers.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py index bf04ae4c..bb0f8fa1 100644 --- a/google/api_core/grpc_helpers.py +++ b/google/api_core/grpc_helpers.py @@ -25,26 +25,19 @@ import google.auth.credentials import google.auth.transport.grpc import google.auth.transport.requests -import google.protobuf -PROTOBUF_VERSION = google.protobuf.__version__ +try: + import grpc_gcp -# The grpcio-gcp package only has support for protobuf < 4 -if PROTOBUF_VERSION[0:2] == "3.": - try: - import grpc_gcp - - warnings.warn( - """Support for grpcio-gcp is deprecated. This feature will be - removed from `google-api-core` after January 1, 2024. If you need to - continue to use this feature, please pin to a specific version of - `google-api-core`.""", - DeprecationWarning, - ) - HAS_GRPC_GCP = True - except ImportError: - HAS_GRPC_GCP = False -else: + warnings.warn( + """Support for grpcio-gcp is deprecated. This feature will be + removed from `google-api-core` after January 1, 2024. If you need to + continue to use this feature, please pin to a specific version of + `google-api-core`.""", + DeprecationWarning, + ) + HAS_GRPC_GCP = True +except ImportError: HAS_GRPC_GCP = False