From 512a65fbfdf0734041bf825fad2d3a3fe94f1e25 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Fri, 21 Feb 2020 12:59:19 +0300 Subject: [PATCH 1/7] feat(api-core): pass retry from result() to done() --- google/api_core/future/polling.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/google/api_core/future/polling.py b/google/api_core/future/polling.py index 6b4c687d..7a219c9d 100644 --- a/google/api_core/future/polling.py +++ b/google/api_core/future/polling.py @@ -78,16 +78,16 @@ def done(self, retry=DEFAULT_RETRY): # pylint: disable=redundant-returns-doc, missing-raises-doc raise NotImplementedError() - def _done_or_raise(self): + def _done_or_raise(self, retry=DEFAULT_RETRY): """Check if the future is done and raise if it's not.""" - if not self.done(): + if not self.done(retry): raise _OperationNotComplete() def running(self): """True if the operation is currently running.""" return not self.done() - def _blocking_poll(self, timeout=None): + def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY): """Poll and wait for the Future to be resolved. Args: @@ -101,13 +101,13 @@ def _blocking_poll(self, timeout=None): retry_ = self._retry.with_deadline(timeout) try: - retry_(self._done_or_raise)() + retry_(self._done_or_raise)(retry) except exceptions.RetryError: raise concurrent.futures.TimeoutError( "Operation did not complete within the designated " "timeout." ) - def result(self, timeout=None): + def result(self, timeout=None, retry=DEFAULT_RETRY): """Get the result of the operation, blocking if necessary. Args: @@ -122,7 +122,7 @@ def result(self, timeout=None): google.api_core.GoogleAPICallError: If the operation errors or if the timeout is reached before the operation completes. """ - self._blocking_poll(timeout=timeout) + self._blocking_poll(timeout=timeout, retry=retry) if self._exception is not None: # pylint: disable=raising-bad-type From a5288b20869ae7853e977b9bea2f56024a0bdaaf Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Fri, 21 Feb 2020 13:23:46 +0300 Subject: [PATCH 2/7] fix tests --- tests/unit/future/test_polling.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/future/test_polling.py b/tests/unit/future/test_polling.py index c67de064..9529e713 100644 --- a/tests/unit/future/test_polling.py +++ b/tests/unit/future/test_polling.py @@ -87,7 +87,7 @@ def __init__(self): self.poll_count = 0 self.event = threading.Event() - def done(self): + def done(self, retry=polling.DEFAULT_RETRY): self.poll_count += 1 self.event.wait() self.set_result(42) @@ -108,7 +108,7 @@ def test_result_with_polling(): class PollingFutureImplTimeout(PollingFutureImplWithPoll): - def done(self): + def done(self, retry=polling.DEFAULT_RETRY): time.sleep(1) return False @@ -130,7 +130,7 @@ def __init__(self, errors): super(PollingFutureImplTransient, self).__init__() self._errors = errors - def done(self): + def done(self, retry=polling.DEFAULT_RETRY): if self._errors: error, self._errors = self._errors[0], self._errors[1:] raise error("testing") From e8c1de448093f092236a21cadb751d2b298c1661 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Wed, 7 Oct 2020 11:16:21 +0300 Subject: [PATCH 3/7] pass retry into done() only if it is not default --- google/api_core/future/polling.py | 6 ++++-- tests/unit/future/test_polling.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/google/api_core/future/polling.py b/google/api_core/future/polling.py index 7a219c9d..142ea30c 100644 --- a/google/api_core/future/polling.py +++ b/google/api_core/future/polling.py @@ -80,7 +80,9 @@ def done(self, retry=DEFAULT_RETRY): def _done_or_raise(self, retry=DEFAULT_RETRY): """Check if the future is done and raise if it's not.""" - if not self.done(retry): + kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry} + + if not self.done(**kwargs): raise _OperationNotComplete() def running(self): @@ -101,7 +103,7 @@ def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY): retry_ = self._retry.with_deadline(timeout) try: - retry_(self._done_or_raise)(retry) + retry_(self._done_or_raise)(retry=retry) except exceptions.RetryError: raise concurrent.futures.TimeoutError( "Operation did not complete within the designated " "timeout." diff --git a/tests/unit/future/test_polling.py b/tests/unit/future/test_polling.py index 9529e713..5e5bc2ee 100644 --- a/tests/unit/future/test_polling.py +++ b/tests/unit/future/test_polling.py @@ -43,6 +43,8 @@ def test_polling_future_constructor(): assert not future.cancelled() assert future.running() assert future.cancel() + with mock.patch.object(future, "done", return_value=True): + future.result() def test_set_result(): From 812a2e516828b61e9faf5b10fed355c59d9e4695 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Tue, 13 Oct 2020 10:58:04 +0300 Subject: [PATCH 4/7] add more unit tests --- tests/unit/future/test_polling.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/unit/future/test_polling.py b/tests/unit/future/test_polling.py index 5e5bc2ee..97a1823a 100644 --- a/tests/unit/future/test_polling.py +++ b/tests/unit/future/test_polling.py @@ -19,7 +19,7 @@ import mock import pytest -from google.api_core import exceptions +from google.api_core import exceptions, retry from google.api_core.future import polling @@ -194,3 +194,29 @@ def test_double_callback_background_thread(): assert future.poll_count == 1 callback.assert_called_once_with(future) callback2.assert_called_once_with(future) + + +class PollingFutureImplWithoutRetry(PollingFutureImpl): + def done(self): + return True + + def result(self): + return True + + +def test_polling_future_without_retry(): + custom_retry = retry.Retry( + predicate=retry.if_exception_type(exceptions.TooManyRequests) + ) + future = PollingFutureImplWithoutRetry() + assert future.done() + assert future.result() + assert future.running() + + with mock.patch.object(future, "done") as done_mock: + future._done_or_raise() + done_mock.assert_called_once_with() + + with mock.patch.object(future, "done") as done_mock: + future._done_or_raise(retry=custom_retry) + done_mock.assert_called_once_with(retry=custom_retry) From 6e411ecc0f23fc00f8a8366672d1f4f70f88d85e Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Tue, 13 Oct 2020 18:04:17 +0300 Subject: [PATCH 5/7] use super() in the overriden result() method --- tests/unit/future/test_polling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/future/test_polling.py b/tests/unit/future/test_polling.py index 97a1823a..b329a243 100644 --- a/tests/unit/future/test_polling.py +++ b/tests/unit/future/test_polling.py @@ -201,7 +201,7 @@ def done(self): return True def result(self): - return True + return super().result() def test_polling_future_without_retry(): @@ -210,8 +210,8 @@ def test_polling_future_without_retry(): ) future = PollingFutureImplWithoutRetry() assert future.done() - assert future.result() assert future.running() + assert future.result() is None with mock.patch.object(future, "done") as done_mock: future._done_or_raise() From 0c17181e0f32c00e7cc12913e76b6149a55440d7 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Tue, 13 Oct 2020 18:19:30 +0300 Subject: [PATCH 6/7] fix super() function call for py-2.x --- tests/unit/future/test_polling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/future/test_polling.py b/tests/unit/future/test_polling.py index b329a243..bbec67aa 100644 --- a/tests/unit/future/test_polling.py +++ b/tests/unit/future/test_polling.py @@ -201,7 +201,7 @@ def done(self): return True def result(self): - return super().result() + return super(PollingFutureImplWithoutRetry, self).result() def test_polling_future_without_retry(): From 2b2147101149eb823e11f04dfac983a825ce108a Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Thu, 15 Oct 2020 12:27:35 +0300 Subject: [PATCH 7/7] add kwargs guard --- google/api_core/future/polling.py | 6 ++++-- tests/unit/future/test_polling.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/google/api_core/future/polling.py b/google/api_core/future/polling.py index 142ea30c..6466838f 100644 --- a/google/api_core/future/polling.py +++ b/google/api_core/future/polling.py @@ -103,7 +103,8 @@ def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY): retry_ = self._retry.with_deadline(timeout) try: - retry_(self._done_or_raise)(retry=retry) + kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry} + retry_(self._done_or_raise)(**kwargs) except exceptions.RetryError: raise concurrent.futures.TimeoutError( "Operation did not complete within the designated " "timeout." @@ -124,7 +125,8 @@ def result(self, timeout=None, retry=DEFAULT_RETRY): google.api_core.GoogleAPICallError: If the operation errors or if the timeout is reached before the operation completes. """ - self._blocking_poll(timeout=timeout, retry=retry) + kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry} + self._blocking_poll(timeout=timeout, **kwargs) if self._exception is not None: # pylint: disable=raising-bad-type diff --git a/tests/unit/future/test_polling.py b/tests/unit/future/test_polling.py index bbec67aa..2381d036 100644 --- a/tests/unit/future/test_polling.py +++ b/tests/unit/future/test_polling.py @@ -203,6 +203,19 @@ def done(self): def result(self): return super(PollingFutureImplWithoutRetry, self).result() + def _blocking_poll(self, timeout): + return super(PollingFutureImplWithoutRetry, self)._blocking_poll( + timeout=timeout + ) + + +class PollingFutureImplWith_done_or_raise(PollingFutureImpl): + def done(self): + return True + + def _done_or_raise(self): + return super(PollingFutureImplWith_done_or_raise, self)._done_or_raise() + def test_polling_future_without_retry(): custom_retry = retry.Retry( @@ -220,3 +233,10 @@ def test_polling_future_without_retry(): with mock.patch.object(future, "done") as done_mock: future._done_or_raise(retry=custom_retry) done_mock.assert_called_once_with(retry=custom_retry) + + +def test_polling_future_with__done_or_raise(): + future = PollingFutureImplWith_done_or_raise() + assert future.done() + assert future.running() + assert future.result() is None