Thanks to visit codestin.com
Credit goes to github.com

Skip to content

feat: make publish futures compatible with concurrent.futures.as_completed() #397

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 15, 2021
Merged

feat: make publish futures compatible with concurrent.futures.as_completed() #397

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
318676a
feat: make futures compatible with as_completed()
plamut Apr 26, 2021
64d0d6c
Fix two unit tests in pre-Python 3.8
plamut Apr 27, 2021
46b1607
Cover missing code line with a test
plamut Apr 28, 2021
bb3dd72
Use double underscore for internal cancelled flag
plamut May 14, 2021
79f11b6
Prefix manager reference with double underscore
plamut May 14, 2021
6c8b501
Remove Future's completed parameter altogether
plamut May 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 26 additions & 139 deletions google/cloud/pubsub_v1/futures.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,13 @@

from __future__ import absolute_import

import threading
import uuid
import concurrent.futures
import warnings

import google.api_core.future
from google.cloud.pubsub_v1.publisher import exceptions


class Future(google.api_core.future.Future):
class Future(concurrent.futures.Future, google.api_core.future.Future):
"""Encapsulation of the asynchronous execution of an action.

This object is returned from asychronous Pub/Sub calls, and is the
Expand All @@ -31,156 +30,44 @@ class Future(google.api_core.future.Future):
methods in this library.

Args:
completed (Optional[Any]): An event, with the same interface as
:class:`threading.Event`. This is provided so that callers
with different concurrency models (e.g. ``threading`` or
``multiprocessing``) can supply an event that is compatible
with that model. The ``wait()`` and ``set()`` methods will be
used. If this argument is not provided, then a new
:class:`threading.Event` will be created and used.
completed (Optional[Any]):
[deprecated] Not used anymore and will be removed in the future.
"""

# This could be a sentinel object or None, but the sentinel object's ID
# can change if the process is forked, and None has the possibility of
# actually being a result.
_SENTINEL = uuid.uuid4()

def __init__(self, completed=None):
self._result = self._SENTINEL
self._exception = self._SENTINEL
self._callbacks = []
if completed is None:
completed = threading.Event()
self._completed = completed

def cancel(self):
"""Actions in Pub/Sub generally may not be canceled.

This method always returns False.
"""
return False
if completed is not None:
msg = (
"Parameter `completed` is not used anymore and will be "
"removed in the future."
)
warnings.warn(msg, category=DeprecationWarning)

def cancelled(self):
"""Actions in Pub/Sub generally may not be canceled.

This method always returns False.
"""
return False
super().__init__()

def running(self):
"""Actions in Pub/Sub generally may not be canceled.
"""Return ``True`` if the associated Pub/Sub action has not yet completed.

Returns:
bool: ``True`` if this method has not yet completed, or
``False`` if it has completed.
Returns: bool:
"""
return not self.done()

def done(self):
"""Return True the future is done, False otherwise.

This still returns True in failure cases; checking :meth:`result` or
:meth:`exception` is the canonical way to assess success or failure.
"""
return self._exception != self._SENTINEL or self._result != self._SENTINEL

def result(self, timeout=None):
"""Resolve the future and return a value where appropriate.

Args:
timeout (Union[int, float]): The number of seconds before this call
times out and raises TimeoutError.

Raises:
concurrent.futures.TimeoutError: If the request times out.
Exception: For undefined exceptions in the underlying
call execution.
"""
# Attempt to get the exception if there is one.
# If there is not one, then we know everything worked, and we can
# return an appropriate value.
err = self.exception(timeout=timeout)
if err is None:
return self._result
raise err

def exception(self, timeout=None):
"""Return the exception raised by the call, if any.

Args:
timeout (Union[int, float]): The number of seconds before this call
times out and raises TimeoutError.

Raises:
concurrent.futures.TimeoutError: If the request times out.

Returns:
Exception: The exception raised by the call, if any.
"""
# Wait until the future is done.
if not self._completed.wait(timeout=timeout):
raise exceptions.TimeoutError("Timed out waiting for result.")

# If the batch completed successfully, this should return None.
if self._result != self._SENTINEL:
return None

# Okay, this batch had an error; this should return it.
return self._exception

def add_done_callback(self, callback):
"""Attach the provided callable to the future.

The provided function is called, with this future as its only argument,
when the future finishes running.

Args:
callback (Callable): The function to call.

Returns:
None
"""
if self.done():
return callback(self)
self._callbacks.append(callback)
def set_running_or_notify_cancel(self):
raise NotImplementedError(
"Only used by executors from `concurrent.futures` package."
)

def set_result(self, result):
"""Set the result of the future to the provided result.
"""Set the return value of work associated with the future.

Args:
result (Any): The result
Do not use this method, it should only be used internally by the library and its
unit tests.
"""
# Sanity check: A future can only complete once.
if self.done():
raise RuntimeError("set_result can only be called once.")

# Set the result and trigger the future.
self._result = result
self._trigger()
return super().set_result(result=result)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need this override?


def set_exception(self, exception):
"""Set the result of the future to the given exception.

Args:
exception (:exc:`Exception`): The exception raised.
"""
# Sanity check: A future can only complete once.
if self.done():
raise RuntimeError("set_exception can only be called once.")

# Set the exception and trigger the future.
self._exception = exception
self._trigger()

def _trigger(self):
"""Trigger all callbacks registered to this Future.

This method is called internally by the batch once the batch
completes.
"""Set the result of the future as being the given exception.

Args:
message_id (str): The message ID, as a string.
Do not use this method, it should only be used internally by the library and its
unit tests.
"""
self._completed.set()
for callback in self._callbacks:
callback(self)
return super().set_exception(exception=exception)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need this override?

2 changes: 1 addition & 1 deletion google/cloud/pubsub_v1/publisher/_batch/thread.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,7 @@ def publish(self, message):

# Track the future on this batch (so that the result of the
# future can be set).
future = futures.Future(completed=threading.Event())
future = futures.Future()
self._futures.append(future)

# Try to commit, but it must be **without** the lock held, since
Expand Down
22 changes: 15 additions & 7 deletions google/cloud/pubsub_v1/publisher/futures.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,20 @@ class Future(futures.Future):
ID, unless an error occurs.
"""

def cancel(self):
"""Actions in Pub/Sub generally may not be canceled.

This method always returns ``False``.
"""
return False

def cancelled(self):
"""Actions in Pub/Sub generally may not be canceled.

This method always returns ``False``.
"""
return False

def result(self, timeout=None):
"""Return the message ID or raise an exception.

Expand All @@ -43,10 +57,4 @@ def result(self, timeout=None):
Exception: For undefined exceptions in the underlying
call execution.
"""
# Attempt to get the exception if there is one.
# If there is not one, then we know everything worked, and we can
# return an appropriate value.
err = self.exception(timeout=timeout)
if err is None:
return self._result
raise err
return super().result(timeout=timeout)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need this override?

2 changes: 2 additions & 0 deletions google/cloud/pubsub_v1/subscriber/futures.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ def cancel(self):
"""Stops pulling messages and shutdowns the background thread consuming
messages.
"""
# NOTE: We circumvent the base future's self._state to track the cancellation
# state, as this state has different meaning with streaming pull futures.
self._cancelled = True
return self._manager.close()

Expand Down
8 changes: 8 additions & 0 deletions tests/unit/pubsub_v1/publisher/test_futures_publisher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,14 @@


class TestFuture(object):
def test_cancel(self):
future = futures.Future()
assert future.cancel() is False

def test_cancelled(self):
future = futures.Future()
assert future.cancelled() is False

def test_result_on_success(self):
future = futures.Future()
future.set_result("570307942214048")
Expand Down
Loading