From bf74b4a740fb3662f68864056228c443b823f36e Mon Sep 17 00:00:00 2001 From: abbrowne126 <81702808+abbrowne126@users.noreply.github.com> Date: Fri, 23 May 2025 12:21:37 -0400 Subject: [PATCH 1/9] fix: add stop() call to BackgroundConsumer failures due to exceptions --- google/api_core/bidi.py | 1 + tests/unit/test_bidi.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index b002b409..0ff6dccd 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -684,6 +684,7 @@ def _thread_main(self, ready): exc, ) + self.stop() _LOGGER.info("%s exiting", _BIDIRECTIONAL_CONSUMER_NAME) def start(self): diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index 0e7b018c..43d783d6 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -918,3 +918,23 @@ def test_stop_error_logs(self, caplog): error_logs = [r.message for r in caplog.records if r.levelname == "ERROR"] assert not error_logs, f"Found unexpected ERROR logs: {error_logs}" bidi_rpc.is_active = False + + def test_fatal_exceptions_will_shutdown_consumer(self, caplog): + """ + https://github.com/googleapis/python-api-core/issues/820 + Exceptions thrown in the BackgroundConsumer that + lead to the consumer halting should also stop the thread and rpc. + """ + caplog.set_level(logging.DEBUG) + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + on_response = mock.Mock(spec=["__call__"]) + + bidi_rpc.open.side_effect = ValueError() + + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + + consumer.start() + + # We want to make sure that close is called, which will surface the error to the caller. + bidi_rpc.close.assert_called_once() From 8a4d671fe7bac90cdf81bff419c3e39bfd66e226 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 23 May 2025 16:24:24 +0000 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/unit/test_bidi.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index 43d783d6..c6924630 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -920,21 +920,21 @@ def test_stop_error_logs(self, caplog): bidi_rpc.is_active = False def test_fatal_exceptions_will_shutdown_consumer(self, caplog): - """ - https://github.com/googleapis/python-api-core/issues/820 - Exceptions thrown in the BackgroundConsumer that - lead to the consumer halting should also stop the thread and rpc. - """ - caplog.set_level(logging.DEBUG) - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) - bidi_rpc.is_active = True - on_response = mock.Mock(spec=["__call__"]) + """ + https://github.com/googleapis/python-api-core/issues/820 + Exceptions thrown in the BackgroundConsumer that + lead to the consumer halting should also stop the thread and rpc. + """ + caplog.set_level(logging.DEBUG) + bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc.is_active = True + on_response = mock.Mock(spec=["__call__"]) - bidi_rpc.open.side_effect = ValueError() + bidi_rpc.open.side_effect = ValueError() - consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) - consumer.start() + consumer.start() - # We want to make sure that close is called, which will surface the error to the caller. - bidi_rpc.close.assert_called_once() + # We want to make sure that close is called, which will surface the error to the caller. + bidi_rpc.close.assert_called_once() From c18475cfdee6df307ec2afa2d567b03fd8ccd6bc Mon Sep 17 00:00:00 2001 From: abbrowne126 <81702808+abbrowne126@users.noreply.github.com> Date: Fri, 23 May 2025 12:39:32 -0400 Subject: [PATCH 3/9] move stop call to within the exception clauses --- google/api_core/bidi.py | 3 ++- tests/unit/test_bidi.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index 0ff6dccd..a22ef558 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -676,6 +676,7 @@ def _thread_main(self, ready): exc, exc_info=True, ) + self.stop() except Exception as exc: _LOGGER.exception( @@ -683,8 +684,8 @@ def _thread_main(self, ready): _BIDIRECTIONAL_CONSUMER_NAME, exc, ) + self.stop() - self.stop() _LOGGER.info("%s exiting", _BIDIRECTIONAL_CONSUMER_NAME) def start(self): diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index 43d783d6..9fabdc52 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -926,7 +926,7 @@ def test_fatal_exceptions_will_shutdown_consumer(self, caplog): lead to the consumer halting should also stop the thread and rpc. """ caplog.set_level(logging.DEBUG) - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc = mock.create_autospec(bidi.ResumableBidiRpc, instance=True) bidi_rpc.is_active = True on_response = mock.Mock(spec=["__call__"]) From 8a9a7d860dac94b8d0872473c55bfc498d03c1fc Mon Sep 17 00:00:00 2001 From: abbrowne126 <81702808+abbrowne126@users.noreply.github.com> Date: Fri, 23 May 2025 12:43:07 -0400 Subject: [PATCH 4/9] Update test_bidi.py --- tests/unit/test_bidi.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index c6924630..da22b0ed 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -926,7 +926,7 @@ def test_fatal_exceptions_will_shutdown_consumer(self, caplog): lead to the consumer halting should also stop the thread and rpc. """ caplog.set_level(logging.DEBUG) - bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True) + bidi_rpc = mock.create_autospec(bidi.ResumableBidiRpc, instance=True) bidi_rpc.is_active = True on_response = mock.Mock(spec=["__call__"]) @@ -936,5 +936,8 @@ def test_fatal_exceptions_will_shutdown_consumer(self, caplog): consumer.start() + # let the background thread run for a while before exiting + time.sleep(0.1) + # We want to make sure that close is called, which will surface the error to the caller. bidi_rpc.close.assert_called_once() From 2065df0a8a4e0e0e6b639d7f4d3c11d4e23510b6 Mon Sep 17 00:00:00 2001 From: abbrowne126 <81702808+abbrowne126@users.noreply.github.com> Date: Fri, 30 May 2025 10:57:45 -0400 Subject: [PATCH 5/9] fix: allow exceptions to be surfaced to the caller of BackgroundConsumer --- google/api_core/bidi.py | 15 +++++++++++---- tests/unit/test_bidi.py | 29 +++++++++++++++++------------ 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index a22ef558..3acc3215 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -624,12 +624,17 @@ def on_response(response): ``open()``ed yet. on_response (Callable[[protobuf.Message], None]): The callback to be called for every response on the stream. + reraise_exceptions (bool): Whether to reraise exceptions during + the lifetime of the consumer, generally those that are not + handled by `BidiRpc`'s `should_recover` or `should_terminate`. + Default `False`. """ - def __init__(self, bidi_rpc, on_response): + def __init__(self, bidi_rpc, on_response, reraise_exceptions=False): self._bidi_rpc = bidi_rpc self._on_response = on_response self._paused = False + self._reraise_exceptions = reraise_exceptions self._wake = threading.Condition() self._thread = None self._operational_lock = threading.Lock() @@ -676,7 +681,8 @@ def _thread_main(self, ready): exc, exc_info=True, ) - self.stop() + if self._reraise_exceptions: + raise except Exception as exc: _LOGGER.exception( @@ -684,7 +690,8 @@ def _thread_main(self, ready): _BIDIRECTIONAL_CONSUMER_NAME, exc, ) - self.stop() + if self._reraise_exceptions: + raise _LOGGER.info("%s exiting", _BIDIRECTIONAL_CONSUMER_NAME) @@ -696,8 +703,8 @@ def start(self): name=_BIDIRECTIONAL_CONSUMER_NAME, target=self._thread_main, args=(ready,), + daemon=True, ) - thread.daemon = True thread.start() # Other parts of the code rely on `thread.is_alive` which # isn't sufficient to know if a thread is active, just that it may diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index da22b0ed..d2f70203 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -922,22 +922,27 @@ def test_stop_error_logs(self, caplog): def test_fatal_exceptions_will_shutdown_consumer(self, caplog): """ https://github.com/googleapis/python-api-core/issues/820 - Exceptions thrown in the BackgroundConsumer that - lead to the consumer halting should also stop the thread and rpc. + Exceptions thrown in the BackgroundConsumer not caught by `should_recover` / `should_terminate` + on the RPC should be bubbled back to the caller if `reraise_exceptions` is `True`. """ caplog.set_level(logging.DEBUG) - bidi_rpc = mock.create_autospec(bidi.ResumableBidiRpc, instance=True) - bidi_rpc.is_active = True - on_response = mock.Mock(spec=["__call__"]) - bidi_rpc.open.side_effect = ValueError() + for fatal_exception in ( + ValueError("some non-api error"), + exceptions.PermissionDenied("some api error"), + ): + bidi_rpc = mock.create_autospec(bidi.ResumableBidiRpc, instance=True) + bidi_rpc.is_active = True + on_response = mock.Mock(spec=["__call__"]) - consumer = bidi.BackgroundConsumer(bidi_rpc, on_response) + bidi_rpc.open.side_effect = fatal_exception - consumer.start() + consumer = bidi.BackgroundConsumer( + bidi_rpc, on_response, reraise_exceptions=True + ) - # let the background thread run for a while before exiting - time.sleep(0.1) + with pytest.raises(type(fatal_exception)): + consumer.start() - # We want to make sure that close is called, which will surface the error to the caller. - bidi_rpc.close.assert_called_once() + # let the background thread run for a while before exiting + time.sleep(0.1) From 48b8f695e2b8e87f00abf19e784daadd1712c2c5 Mon Sep 17 00:00:00 2001 From: abbrowne126 <81702808+abbrowne126@users.noreply.github.com> Date: Fri, 30 May 2025 10:58:47 -0400 Subject: [PATCH 6/9] Update bidi.py --- google/api_core/bidi.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index 3acc3215..3b896705 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -715,7 +715,11 @@ def start(self): _LOGGER.debug("Started helper thread %s", thread.name) def stop(self): - """Stop consuming the stream and shutdown the background thread.""" + """Stop consuming the stream and shutdown the background thread. + + WARNING: Cannot be called within `_thread_main`, since it is not + possible to join a thread to itself. + """ with self._operational_lock: self._bidi_rpc.close() From d220fa630c78a888e36c4ab0b4cc028fe1b7691b Mon Sep 17 00:00:00 2001 From: abbrowne126 <81702808+abbrowne126@users.noreply.github.com> Date: Fri, 30 May 2025 11:15:27 -0400 Subject: [PATCH 7/9] fix: allow BackgroundConsumer caller to pass `on_fatal_exception` to be informed of fatal processing errors --- google/api_core/bidi.py | 19 +++++++++---------- tests/unit/test_bidi.py | 16 +++++++++------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index 3b896705..f4102930 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -624,17 +624,15 @@ def on_response(response): ``open()``ed yet. on_response (Callable[[protobuf.Message], None]): The callback to be called for every response on the stream. - reraise_exceptions (bool): Whether to reraise exceptions during - the lifetime of the consumer, generally those that are not - handled by `BidiRpc`'s `should_recover` or `should_terminate`. - Default `False`. + on_fatal_exception (Callable[[Exception], None]): The callback to + be called on fatal errors during consumption. Default None. """ - def __init__(self, bidi_rpc, on_response, reraise_exceptions=False): + def __init__(self, bidi_rpc, on_response, on_fatal_exception=None): self._bidi_rpc = bidi_rpc self._on_response = on_response self._paused = False - self._reraise_exceptions = reraise_exceptions + self._on_fatal_exception = on_fatal_exception self._wake = threading.Condition() self._thread = None self._operational_lock = threading.Lock() @@ -681,8 +679,8 @@ def _thread_main(self, ready): exc, exc_info=True, ) - if self._reraise_exceptions: - raise + if self._on_fatal_exception is not None: + self._on_fatal_exception(exc) except Exception as exc: _LOGGER.exception( @@ -690,8 +688,8 @@ def _thread_main(self, ready): _BIDIRECTIONAL_CONSUMER_NAME, exc, ) - if self._reraise_exceptions: - raise + if self._on_fatal_exception is not None: + self._on_fatal_exception(exc) _LOGGER.info("%s exiting", _BIDIRECTIONAL_CONSUMER_NAME) @@ -734,6 +732,7 @@ def stop(self): self._thread = None self._on_response = None + self._on_fatal_exception = None @property def is_active(self): diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index d2f70203..d122f870 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -919,11 +919,11 @@ def test_stop_error_logs(self, caplog): assert not error_logs, f"Found unexpected ERROR logs: {error_logs}" bidi_rpc.is_active = False - def test_fatal_exceptions_will_shutdown_consumer(self, caplog): + def test_fatal_exceptions_can_inform_consumer(self, caplog): """ https://github.com/googleapis/python-api-core/issues/820 Exceptions thrown in the BackgroundConsumer not caught by `should_recover` / `should_terminate` - on the RPC should be bubbled back to the caller if `reraise_exceptions` is `True`. + on the RPC should be bubbled back to the caller through `on_fatal_exception` if passed. """ caplog.set_level(logging.DEBUG) @@ -935,14 +935,16 @@ def test_fatal_exceptions_will_shutdown_consumer(self, caplog): bidi_rpc.is_active = True on_response = mock.Mock(spec=["__call__"]) + on_fatal_exception = mock.Mock(spec=["__call__"]) + bidi_rpc.open.side_effect = fatal_exception consumer = bidi.BackgroundConsumer( - bidi_rpc, on_response, reraise_exceptions=True + bidi_rpc, on_response, on_fatal_exception ) - with pytest.raises(type(fatal_exception)): - consumer.start() + consumer.start() + # let the background thread run for a while before exiting + time.sleep(0.1) - # let the background thread run for a while before exiting - time.sleep(0.1) + on_fatal_exception.assert_called_once_with(fatal_exception) From 7fc87c05e4f500a82d9e9493b9f6b91c535973ec Mon Sep 17 00:00:00 2001 From: Andrew Browne <81702808+abbrowne126@users.noreply.github.com> Date: Fri, 30 May 2025 12:44:21 -0400 Subject: [PATCH 8/9] Update tests/unit/test_bidi.py Co-authored-by: Anthonios Partheniou --- tests/unit/test_bidi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py index d122f870..7640367c 100644 --- a/tests/unit/test_bidi.py +++ b/tests/unit/test_bidi.py @@ -923,7 +923,7 @@ def test_fatal_exceptions_can_inform_consumer(self, caplog): """ https://github.com/googleapis/python-api-core/issues/820 Exceptions thrown in the BackgroundConsumer not caught by `should_recover` / `should_terminate` - on the RPC should be bubbled back to the caller through `on_fatal_exception` if passed. + on the RPC should be bubbled back to the caller through `on_fatal_exception`, if passed. """ caplog.set_level(logging.DEBUG) From b9e7edcfa55c7e1a00853aa1329fed1b98fdc925 Mon Sep 17 00:00:00 2001 From: abbrowne126 <81702808+abbrowne126@users.noreply.github.com> Date: Fri, 30 May 2025 12:44:37 -0400 Subject: [PATCH 9/9] Update bidi.py --- google/api_core/bidi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py index f4102930..bed4c70e 100644 --- a/google/api_core/bidi.py +++ b/google/api_core/bidi.py @@ -715,7 +715,7 @@ def start(self): def stop(self): """Stop consuming the stream and shutdown the background thread. - WARNING: Cannot be called within `_thread_main`, since it is not + NOTE: Cannot be called within `_thread_main`, since it is not possible to join a thread to itself. """ with self._operational_lock: