From 8d8bc5150ee5543b4aeb2c271da034a5305d1436 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 16:30:03 +0000 Subject: [PATCH 1/4] docs: fix broken links in multiprocessing.rst (#195) gcr.io/repo-automation-bots/owlbot-python:latest@sha256:0856ca711da1fd5ec9d6d7da6c50aa0bbf550fb94acb47b55159a640791987bf --- .github/.OwlBot.lock.yaml | 2 +- docs/multiprocessing.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 864c1765..127c2cdf 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:4c981a6b6f2b8914a448d7b3a01688365be03e3ed26dfee399a6aa77fb112eaa + digest: sha256:0856ca711da1fd5ec9d6d7da6c50aa0bbf550fb94acb47b55159a640791987bf diff --git a/docs/multiprocessing.rst b/docs/multiprocessing.rst index 1cb29d4c..536d17b2 100644 --- a/docs/multiprocessing.rst +++ b/docs/multiprocessing.rst @@ -1,7 +1,7 @@ .. note:: - Because this client uses :mod:`grpcio` library, it is safe to + Because this client uses :mod:`grpc` library, it is safe to share instances across threads. In multiprocessing scenarios, the best practice is to create client instances *after* the invocation of - :func:`os.fork` by :class:`multiprocessing.Pool` or + :func:`os.fork` by :class:`multiprocessing.pool.Pool` or :class:`multiprocessing.Process`. From 7337c6b123735fb9ae5d0e54b4399275719c020c Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 28 May 2021 16:50:03 +0000 Subject: [PATCH 2/4] chore(python): add protobuf to intersphinx mapping (#198) Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:c66ba3c8d7bc8566f47df841f98cd0097b28fff0b1864c86f5817f4c8c3e8600 --- .github/.OwlBot.lock.yaml | 2 +- docs/conf.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 127c2cdf..da616c91 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:0856ca711da1fd5ec9d6d7da6c50aa0bbf550fb94acb47b55159a640791987bf + digest: sha256:c66ba3c8d7bc8566f47df841f98cd0097b28fff0b1864c86f5817f4c8c3e8600 diff --git a/docs/conf.py b/docs/conf.py index 9ec04520..1e73c44b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -366,6 +366,7 @@ ), "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), + "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), } From a4219137a5bfcf2a6f44780ecdbf475c1129e461 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Wed, 2 Jun 2021 12:24:38 -0400 Subject: [PATCH 3/4] =?UTF-8?q?feat:=20HTTPIterator=20now=20accepts=20a=20?= =?UTF-8?q?page=5Fsize=20parameter=20to=20control=20page=20=E2=80=A6=20(#1?= =?UTF-8?q?97)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- google/api_core/page_iterator.py | 19 ++++++++-- tests/unit/test_page_iterator.py | 62 ++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/google/api_core/page_iterator.py b/google/api_core/page_iterator.py index 11a92d38..fff3b556 100644 --- a/google/api_core/page_iterator.py +++ b/google/api_core/page_iterator.py @@ -179,7 +179,7 @@ def __init__( single item. """ self.max_results = max_results - """int: The maximum number of results to fetch.""" + """int: The maximum number of results to fetch""" # The attributes below will change over the life of the iterator. self.page_number = 0 @@ -298,7 +298,8 @@ class HTTPIterator(Iterator): can be found. page_token (str): A token identifying a page in a result set to start fetching results from. - max_results (int): The maximum number of results to fetch. + page_size (int): The maximum number of results to fetch per page + max_results (int): The maximum number of results to fetch extra_params (dict): Extra query string parameters for the API call. page_start (Callable[ @@ -329,6 +330,7 @@ def __init__( item_to_value, items_key=_DEFAULT_ITEMS_KEY, page_token=None, + page_size=None, max_results=None, extra_params=None, page_start=_do_nothing_page_start, @@ -341,6 +343,7 @@ def __init__( self.path = path self._items_key = items_key self.extra_params = extra_params + self._page_size = page_size self._page_start = page_start self._next_token = next_token # Verify inputs / provide defaults. @@ -399,8 +402,18 @@ def _get_query_params(self): result = {} if self.next_page_token is not None: result[self._PAGE_TOKEN] = self.next_page_token + + page_size = None if self.max_results is not None: - result[self._MAX_RESULTS] = self.max_results - self.num_results + page_size = self.max_results - self.num_results + if self._page_size is not None: + page_size = min(page_size, self._page_size) + elif self._page_size is not None: + page_size = self._page_size + + if page_size is not None: + result[self._MAX_RESULTS] = page_size + result.update(self.extra_params) return result diff --git a/tests/unit/test_page_iterator.py b/tests/unit/test_page_iterator.py index 2bf74249..83595376 100644 --- a/tests/unit/test_page_iterator.py +++ b/tests/unit/test_page_iterator.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import math import types import mock @@ -235,6 +236,7 @@ def test_constructor(self): assert iterator.page_number == 0 assert iterator.next_page_token is None assert iterator.num_results == 0 + assert iterator._page_size is None def test_constructor_w_extra_param_collision(self): extra_params = {"pageToken": "val"} @@ -432,6 +434,66 @@ def test__get_next_page_bad_http_method(self): with pytest.raises(ValueError): iterator._get_next_page_response() + @pytest.mark.parametrize( + "page_size,max_results,pages", + [(3, None, False), (3, 8, False), (3, None, True), (3, 8, True)]) + def test_page_size_items(self, page_size, max_results, pages): + path = "/foo" + NITEMS = 10 + + n = [0] # blast you python 2! + + def api_request(*args, **kw): + assert not args + query_params = dict( + maxResults=( + page_size if max_results is None + else min(page_size, max_results - n[0])) + ) + if n[0]: + query_params.update(pageToken='test') + assert kw == {'method': 'GET', 'path': '/foo', + 'query_params': query_params} + n_items = min(kw['query_params']['maxResults'], NITEMS - n[0]) + items = [dict(name=str(i + n[0])) for i in range(n_items)] + n[0] += n_items + result = dict(items=items) + if n[0] < NITEMS: + result.update(nextPageToken='test') + return result + + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, + api_request, + path=path, + item_to_value=page_iterator._item_to_value_identity, + page_size=page_size, + max_results=max_results, + ) + + assert iterator.num_results == 0 + + n_results = max_results if max_results is not None else NITEMS + if pages: + items_iter = iter(iterator.pages) + npages = int(math.ceil(float(n_results) / page_size)) + for ipage in range(npages): + assert ( + list(six.next(items_iter)) == [ + dict(name=str(i)) + for i in range(ipage * page_size, + min((ipage + 1) * page_size, n_results), + ) + ]) + else: + items_iter = iter(iterator) + for i in range(n_results): + assert six.next(items_iter) == dict(name=str(i)) + assert iterator.num_results == i + 1 + + with pytest.raises(StopIteration): + six.next(items_iter) + class TestGRPCIterator(object): def test_constructor(self): From 641fbbf95c4cf72e48e2a58d563e41b2a1787bbf Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 3 Jun 2021 07:40:35 -0400 Subject: [PATCH 4/4] chore: release 1.29.0 (#199) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 12 ++++++++++++ google/api_core/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93deb48a..6373aae7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ [1]: https://pypi.org/project/google-api-core/#history +## [1.29.0](https://www.github.com/googleapis/python-api-core/compare/v1.28.0...v1.29.0) (2021-06-02) + + +### Features + +* HTTPIterator now accepts a page_size parameter to control page … ([#197](https://www.github.com/googleapis/python-api-core/issues/197)) ([a421913](https://www.github.com/googleapis/python-api-core/commit/a4219137a5bfcf2a6f44780ecdbf475c1129e461)) + + +### Documentation + +* fix broken links in multiprocessing.rst ([#195](https://www.github.com/googleapis/python-api-core/issues/195)) ([8d8bc51](https://www.github.com/googleapis/python-api-core/commit/8d8bc5150ee5543b4aeb2c271da034a5305d1436)) + ## [1.28.0](https://www.github.com/googleapis/python-api-core/compare/v1.27.0...v1.28.0) (2021-05-20) diff --git a/google/api_core/version.py b/google/api_core/version.py index 39a86a1a..e5c10108 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.28.0" +__version__ = "1.29.0"