diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index a3da1b0d..ec696b55 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:3e3800bb100af5d7f9e810d48212b37812c1856d20ffeafb99ebe66461b61fc7 -# created: 2023-08-02T10:53:29.114535628Z + digest: sha256:30470597773378105e239b59fce8eb27cc97375580d592699206d17d117143d0 +# created: 2023-11-03T00:57:07.335914631Z diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e97d89e4..221806ce 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -28,7 +28,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.10" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/.gitignore b/.gitignore index 99c3a144..168b201f 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ docs.metadata # Virtual environment env/ +venv/ # Test logs coverage.xml diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 029bd342..16170d0c 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -113,30 +113,30 @@ commonmark==0.9.1 \ --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 # via rich -cryptography==41.0.3 \ - --hash=sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306 \ - --hash=sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84 \ - --hash=sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47 \ - --hash=sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d \ - --hash=sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116 \ - --hash=sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207 \ - --hash=sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81 \ - --hash=sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087 \ - --hash=sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd \ - --hash=sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507 \ - --hash=sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858 \ - --hash=sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae \ - --hash=sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34 \ - --hash=sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906 \ - --hash=sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd \ - --hash=sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922 \ - --hash=sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7 \ - --hash=sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4 \ - --hash=sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574 \ - --hash=sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1 \ - --hash=sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c \ - --hash=sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e \ - --hash=sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de +cryptography==41.0.4 \ + --hash=sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67 \ + --hash=sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311 \ + --hash=sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8 \ + --hash=sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13 \ + --hash=sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143 \ + --hash=sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f \ + --hash=sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829 \ + --hash=sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd \ + --hash=sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397 \ + --hash=sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac \ + --hash=sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d \ + --hash=sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a \ + --hash=sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839 \ + --hash=sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e \ + --hash=sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6 \ + --hash=sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9 \ + --hash=sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860 \ + --hash=sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca \ + --hash=sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91 \ + --hash=sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d \ + --hash=sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714 \ + --hash=sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb \ + --hash=sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f # via # gcp-releasetool # secretstorage @@ -382,6 +382,7 @@ protobuf==3.20.3 \ # gcp-docuploader # gcp-releasetool # google-api-core + # googleapis-common-protos pyasn1==0.4.8 \ --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba @@ -466,9 +467,9 @@ typing-extensions==4.4.0 \ --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e # via -r requirements.in -urllib3==1.26.12 \ - --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \ - --hash=sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997 +urllib3==1.26.18 \ + --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \ + --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0 # via # requests # twine diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 19409cbd..6a8e1695 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: end-of-file-fixer - id: check-yaml - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/pycqa/flake8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 93c3848c..7b0dacfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.13.0](https://github.com/googleapis/python-api-core/compare/v2.12.0...v2.13.0) (2023-11-03) + + +### Features + +* Add caching to routing header calculation ([#526](https://github.com/googleapis/python-api-core/issues/526)) ([6251eab](https://github.com/googleapis/python-api-core/commit/6251eab3fca5f7e509cb9b6e476ce1184094b711)) + + +### Bug Fixes + +* Add warning to retry target to avoid incorrect usage ([#543](https://github.com/googleapis/python-api-core/issues/543)) ([bfb40e6](https://github.com/googleapis/python-api-core/commit/bfb40e6929ef47be7a6464d2f1e0d06595736b8d)) +* Drop usage of distutils ([#541](https://github.com/googleapis/python-api-core/issues/541)) ([4bd9e10](https://github.com/googleapis/python-api-core/commit/4bd9e10f20eea227c88e3e1496010cca6dd8a270)) +* Ensure exception is available when BackgroundConsumer open stream fails ([#357](https://github.com/googleapis/python-api-core/issues/357)) ([405272c](https://github.com/googleapis/python-api-core/commit/405272c05f8c6d20e242c6172b01f78f0fd3bf32)) + ## [2.12.0](https://github.com/googleapis/python-api-core/compare/v2.11.1...v2.12.0) (2023-09-07) diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index 57f5f9dd..74abc495 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -92,10 +92,7 @@ def _is_active(self): # Note: there is a possibility that this starts *before* the call # property is set. So we have to check if self.call is set before # seeing if it's active. - if self.call is not None and not self.call.is_active(): - return False - else: - return True + return self.call is not None and self.call.is_active() def __iter__(self): if self._initial_request is not None: @@ -265,6 +262,10 @@ def add_done_callback(self, callback): self._callbacks.append(callback) def _on_call_done(self, future): + # This occurs when the RPC errors or is successfully terminated. + # Note that grpc's "future" here can also be a grpc.RpcError. + # See note in https://github.com/grpc/grpc/issues/10885#issuecomment-302651331 + # that `grpc.RpcError` is also `grpc.call`. for callback in self._callbacks: callback(future) @@ -276,7 +277,13 @@ def open(self): request_generator = _RequestQueueGenerator( self._request_queue, initial_request=self._initial_request ) - call = self._start_rpc(iter(request_generator), metadata=self._rpc_metadata) + try: + call = self._start_rpc(iter(request_generator), metadata=self._rpc_metadata) + except exceptions.GoogleAPICallError as exc: + # The original `grpc.RpcError` (which is usually also a `grpc.Call`) is + # available from the ``response`` property on the mapped exception. + self._on_call_done(exc.response) + raise request_generator.call = call diff --git a/google/api_core/gapic_v1/routing_header.py b/google/api_core/gapic_v1/routing_header.py index 28b13abf..c0c6f648 100644 --- a/google/api_core/gapic_v1/routing_header.py +++ b/google/api_core/gapic_v1/routing_header.py @@ -20,17 +20,22 @@ Generally, these headers are specified as gRPC metadata. """ +import functools from enum import Enum from urllib.parse import urlencode ROUTING_METADATA_KEY = "x-goog-request-params" +# This is the value for the `maxsize` argument of @functools.lru_cache +# https://docs.python.org/3/library/functools.html#functools.lru_cache +# This represents the number of recent function calls to store. +ROUTING_PARAM_CACHE_SIZE = 32 def to_routing_header(params, qualified_enums=True): """Returns a routing header string for the given request parameters. Args: - params (Mapping[str, Any]): A dictionary containing the request + params (Mapping[str, str | bytes | Enum]): A dictionary containing the request parameters used for routing. qualified_enums (bool): Whether to represent enum values as their type-qualified symbol names instead of as their @@ -38,19 +43,11 @@ def to_routing_header(params, qualified_enums=True): Returns: str: The routing header string. - """ + tuples = params.items() if isinstance(params, dict) else params if not qualified_enums: - if isinstance(params, dict): - tuples = params.items() - else: - tuples = params - params = [(x[0], x[1].name) if isinstance(x[1], Enum) else x for x in tuples] - return urlencode( - params, - # Per Google API policy (go/api-url-encoding), / is not encoded. - safe="/", - ) + tuples = [(x[0], x[1].name) if isinstance(x[1], Enum) else x for x in tuples] + return "&".join([_urlencode_param(*t) for t in tuples]) def to_grpc_metadata(params, qualified_enums=True): @@ -58,7 +55,7 @@ def to_grpc_metadata(params, qualified_enums=True): request parameters. Args: - params (Mapping[str, Any]): A dictionary containing the request + params (Mapping[str, str | bytes | Enum]): A dictionary containing the request parameters used for routing. qualified_enums (bool): Whether to represent enum values as their type-qualified symbol names instead of as their @@ -69,3 +66,22 @@ def to_grpc_metadata(params, qualified_enums=True): and value. """ return (ROUTING_METADATA_KEY, to_routing_header(params, qualified_enums)) + + +# use caching to avoid repeated computation +@functools.lru_cache(maxsize=ROUTING_PARAM_CACHE_SIZE) +def _urlencode_param(key, value): + """Cacheable wrapper over urlencode + + Args: + key (str): The key of the parameter to encode. + value (str | bytes | Enum): The value of the parameter to encode. + + Returns: + str: The encoded parameter. + """ + return urlencode( + {key: value}, + # Per Google API policy (go/api-url-encoding), / is not encoded. + safe="/", + ) diff --git a/google/api_core/operations_v1/abstract_operations_client.py b/google/api_core/operations_v1/abstract_operations_client.py index 714c2aae..38f532af 100644 --- a/google/api_core/operations_v1/abstract_operations_client.py +++ b/google/api_core/operations_v1/abstract_operations_client.py @@ -14,7 +14,6 @@ # limitations under the License. # from collections import OrderedDict -from distutils import util import os import re from typing import Dict, Optional, Sequence, Tuple, Type, Union @@ -294,13 +293,16 @@ def __init__( client_options = client_options_lib.ClientOptions() # Create SSL credentials for mutual TLS if needed. - use_client_cert = bool( - util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false")) - ) - + use_client_cert = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + if use_client_cert not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) client_cert_source_func = None is_mtls = False - if use_client_cert: + if use_client_cert == "true": if client_options.client_cert_source: is_mtls = True client_cert_source_func = client_options.client_cert_source diff --git a/google/api_core/retry.py b/google/api_core/retry.py index 84b5d0fe..08f8209e 100644 --- a/google/api_core/retry.py +++ b/google/api_core/retry.py @@ -62,6 +62,8 @@ def check_if_exists(): import random import sys import time +import inspect +import warnings from typing import Any, Callable, TypeVar, TYPE_CHECKING import requests.exceptions @@ -84,6 +86,7 @@ def check_if_exists(): _DEFAULT_MAXIMUM_DELAY = 60.0 # seconds _DEFAULT_DELAY_MULTIPLIER = 2.0 _DEFAULT_DEADLINE = 60.0 * 2.0 # seconds +_ASYNC_RETRY_WARNING = "Using the synchronous google.api_core.retry.Retry with asynchronous calls may lead to unexpected results. Please use google.api_core.retry_async.AsyncRetry instead." def if_exception_type( @@ -201,7 +204,10 @@ def retry_target( for sleep in sleep_generator: try: - return target() + result = target() + if inspect.isawaitable(result): + warnings.warn(_ASYNC_RETRY_WARNING) + return result # pylint: disable=broad-except # This function explicitly must deal with broad exceptions. diff --git a/google/api_core/version.py b/google/api_core/version.py index 67e043bd..b6000e20 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.12.0" +__version__ = "2.13.0" diff --git a/noxfile.py b/noxfile.py index 84abb498..82ea2669 100644 --- a/noxfile.py +++ b/noxfile.py @@ -232,7 +232,7 @@ def docs(session): ) -@nox.session(python="3.9") +@nox.session(python="3.10") def docfx(session): """Build the docfx yaml files for this library.""" diff --git a/tests/unit/gapic/test_routing_header.py b/tests/unit/gapic/test_routing_header.py index 9d31eb39..2c8c7546 100644 --- a/tests/unit/gapic/test_routing_header.py +++ b/tests/unit/gapic/test_routing_header.py @@ -70,3 +70,34 @@ def test_to_grpc_metadata(): params = [("name", "meep"), ("book.read", "1")] metadata = routing_header.to_grpc_metadata(params) assert metadata == (routing_header.ROUTING_METADATA_KEY, "name=meep&book.read=1") + + +@pytest.mark.parametrize( + "key,value,expected", + [ + ("book.read", "1", "book.read=1"), + ("name", "me/ep", "name=me/ep"), + ("\\", "=", "%5C=%3D"), + (b"hello", "world", "hello=world"), + ("✔️", "✌️", "%E2%9C%94%EF%B8%8F=%E2%9C%8C%EF%B8%8F"), + ], +) +def test__urlencode_param(key, value, expected): + result = routing_header._urlencode_param(key, value) + assert result == expected + + +def test__urlencode_param_caching_performance(): + import time + + key = "key" * 100 + value = "value" * 100 + # time with empty cache + start_time = time.perf_counter() + routing_header._urlencode_param(key, value) + duration = time.perf_counter() - start_time + second_start_time = time.perf_counter() + routing_header._urlencode_param(key, value) + second_duration = time.perf_counter() - second_start_time + # second call should be approximately 10 times faster + assert second_duration < duration / 10 diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index f5e2b72b..84ac9dc5 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -804,6 +804,21 @@ def test_wake_on_error(self): while consumer.is_active: pass + def test_rpc_callback_fires_when_consumer_start_fails(self): + expected_exception = exceptions.InvalidArgument( + "test", response=grpc.StatusCode.INVALID_ARGUMENT + ) + callback = mock.Mock(spec=["__call__"]) + + rpc, _ = make_rpc() + bidi_rpc = bidi.BidiRpc(rpc) + bidi_rpc.add_done_callback(callback) + bidi_rpc._start_rpc.side_effect = expected_exception + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response=None) + consumer.start() + assert callback.call_args.args[0] == grpc.StatusCode.INVALID_ARGUMENT + def test_consumer_expected_error(self, caplog): caplog.set_level(logging.DEBUG) diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index ec27056d..d770f7a2 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -129,6 +129,26 @@ def test_retry_target_non_retryable_error(utcnow, sleep): sleep.assert_not_called() +@mock.patch("asyncio.sleep", autospec=True) +@mock.patch( + "google.api_core.datetime_helpers.utcnow", + return_value=datetime.datetime.min, + autospec=True, +) +@pytest.mark.asyncio +async def test_retry_target_warning_for_retry(utcnow, sleep): + predicate = retry.if_exception_type(ValueError) + target = mock.AsyncMock(spec=["__call__"]) + + with pytest.warns(Warning) as exc_info: + # Note: predicate is just a filler and doesn't affect the test + retry.retry_target(target, predicate, range(10), None) + + assert len(exc_info) == 2 + assert str(exc_info[0].message) == retry._ASYNC_RETRY_WARNING + sleep.assert_not_called() + + @mock.patch("time.sleep", autospec=True) @mock.patch("google.api_core.datetime_helpers.utcnow", autospec=True) def test_retry_target_deadline_exceeded(utcnow, sleep):