Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Resolve Incompatibility with API Core >= 1.17.0 #93

Closed
@crwilcox

Description

@crwilcox

Changes in 1.17.0 are causing an incompatibility with pubsub.

Related Issues:

googleapis/python-api-core#25
#74

Suspected changes that caused the behavior change:

googleapis/python-api-core@2b103b6#diff-d97e81006eaaf29e33270a85aead6aafR146-R148

Repro:


from google.cloud import pubsub_v1
import threading
from time import sleep

import logging

def main():
    logging.basicConfig(level=logging.DEBUG)
    logger = logging.getLogger('sample')
    logger.setLevel(logging.DEBUG)
    PROJECT_ID = "crwilcox-test"
    TOPIC_PATH = "my-topic"
    SUBSCRIPTION_PATH = "my-sub"

    publisher_thread = threading.Thread(target=publisher, args=(PROJECT_ID, TOPIC_PATH))
    publisher_thread.daemon = True
    publisher_thread.start()

    subscriber(PROJECT_ID, SUBSCRIPTION_PATH)

def publisher(
    project_id, topic_name
):
    # TODO project_id = "Your Google Cloud Project ID"
    # TODO topic_name = "Your Pub/Sub topic name"


    publisher = pubsub_v1.PublisherClient()
    # The `topic_path` method creates a fully qualified identifier
    # in the form `projects/{project_id}/topics/{topic_name}`
    topic_path = publisher.topic_path(project_id, topic_name)
    #publisher.create_topic(topic_path)

    for n in range(1, 100000):
        data = u"Message number {}".format(n)
        # Data must be a bytestring
        data = data.encode("utf-8")
        # When you publish a message, the client returns a future.
        future = publisher.publish(topic_path, data=data)
        print(future.result())
        sleep(5)

    print("Published messages.")



def subscriber(
    project_id, subscription_name, timeout=None
):
    # TODO project_id = "Your Google Cloud Project ID"
    # TODO subscription_name = "Your Pub/Sub subscription name"
    # TODO timeout = 5.0  # "How long the subscriber should listen for
    # messages in seconds"

    subscriber = pubsub_v1.SubscriberClient()
    # The `subscription_path` method creates a fully qualified identifier
    # in the form `projects/{project_id}/subscriptions/{subscription_name}`
    subscription_path = subscriber.subscription_path(
        project_id, subscription_name
    )

    def callback(message):
        print("Received message: {}".format(message.data))
        message.ack()

    streaming_pull_future = subscriber.subscribe(
        subscription_path, callback=callback
    )
    print("Listening for messages on {}..\n".format(subscription_path))

    # Wrap subscriber in a 'with' block to automatically call close() when done.
    with subscriber:
        try:
            # When `timeout` is not set, result() will block indefinitely,
            # unless an exception is encountered first.
            streaming_pull_future.result(timeout=timeout)
        except:  # noqa
            streaming_pull_future.cancel()


main()

Result:

Eventually, subscription will fail and stop processing.

The failure exists in bidi on getting the next item via __iter__

DEBUG:google.cloud.pubsub_v1.publisher._batch.thread:gRPC Publish took 0.034303903579711914 seconds.
1162512773129262
DEBUG:google.cloud.pubsub_v1.subscriber._protocol.streaming_pull_manager:Sent request(s) over unary RPC.
DEBUG:google.cloud.pubsub_v1.subscriber._protocol.streaming_pull_manager:Released held message, scheduling callback for it, still on hold 0 (bytes 0).
DEBUG:google.api_core.bidi:waiting for recv.
Received message: b'Message number 117'
DEBUG:google.cloud.pubsub_v1.subscriber._protocol.dispatcher:Handling 1 batched requests
DEBUG:google.cloud.pubsub_v1.subscriber._protocol.streaming_pull_manager:Sent request(s) over unary RPC.
iter except: <_MultiThreadedRendezvous of RPC that terminated with:
        status = StatusCode.DEADLINE_EXCEEDED
        details = "Deadline Exceeded"
        debug_error_string = "{"created":"@1588195797.686297000","description":"Error received from peer ipv4:216.58.217.42:443","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Deadline Exceeded","grpc_status":4}"
>
INFO:google.cloud.pubsub_v1.subscriber._protocol.streaming_pull_manager:Observed non-terminating stream error 504 Deadline Exceeded
INFO:google.cloud.pubsub_v1.subscriber._protocol.streaming_pull_manager:Observed recoverable stream error 504 Deadline Exceeded
DEBUG:google.api_core.bidi:Re-opening stream from gRPC callback.
DEBUG:google.api_core.bidi:Empty queue and inactive call, exiting request generator.
DEBUG:google.cloud.pubsub_v1.subscriber._protocol.leaser:The current deadline value is 10 seconds.
PYTHON_PATH/lib/python3.8/threading.py(890)_bootstrap()
-> self._bootstrap_inner()
  PYTHON_PATH/lib/python3.8/threading.py(932)_bootstrap_inner()
-> self.run()
  PYTHON_PATH/lib/python3.8/threading.py(870)run()
-> self._target(*self._args, **self._kwargs)
  /workspace/googleapis/python-pubsub/venv/lib/python3.8/site-packages/google/api_core/bidi.py(655)_thread_main()
-> response = self._bidi_rpc.recv()
  /workspace/googleapis/python-pubsub/venv/lib/python3.8/site-packages/google/api_core/bidi.py(562)recv()
-> return self._recoverable(self._recv)
  /workspace/googleapis/python-pubsub/venv/lib/python3.8/site-packages/google/api_core/bidi.py(505)_recoverable()
-> return method(*args, **kwargs)
  /workspace/googleapis/python-pubsub/venv/lib/python3.8/site-packages/google/api_core/bidi.py(559)_recv()
-> return next(call)
> /workspace/googleapis/python-pubsub/venv/lib/python3.8/site-packages/google/api_core/grpc_helpers.py(109)next()
-> six.raise_from(exceptions.from_grpc_error(exc), exc)

Metadata

Metadata

Assignees

Labels

api: pubsubIssues related to the googleapis/python-pubsub API.priority: p2Moderately-important priority. Fix may not be included in next release.type: bugError or flaw in code with unintended results or allowing sub-optimal usage patterns.

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions