Closed
Description
-
Context
google-cloud-pubsub (0.32.0) Python 2.7.13 Win10
-
Subscriber console | Fails w/ exception after a random time (from 15 min to 3 h)
> python pubsub_subscribe.py Subscribed to messages on projects/PROJECT_ID/subscriptions/TOPIC Waiting for future... Messages received: 0 + 300 Messages received: 300 + 3000 Traceback (most recent call last): File ".\pubsub_subscribe.py", line 58, in <module> subscription.future.result() File "C:\Python27\lib\site-packages\google\cloud\pubsub_v1\futures.py", line 114, in result raise err google.api_core.exceptions.Unknown: None Stream removed
-
Publisher console | Completes as expected
> python pubsub_publish.py Messages sent: 0 + 300 Messages sent: 300 + 3000 Messages sent: 3300 + 30000 Messages sent: 33300 + 30 Messages sent: 33330 + 5 Messages sent: 33335 + 3 Messages sent: 33338 + 2 Messages sent: 33340 + 30000 Messages sent: 63340 + 3000 Messages sent: 66340 + 300 Total messages sent: 66640 -
Initialization
- GCP project
GOOGLE_CLOUD_PROJECT - Pub/Sub API enabled
- Existing topic
PUBSUB_TOPIC - Environment variables
GOOGLE_CLOUD_PROJECT&PUBSUB_TOPIC
- GCP project
-
Subscriber sample code
<pubsub_subscribe.py>import os from socket import gethostname from time import sleep from google.cloud import pubsub from google.api_core.exceptions import AlreadyExists PROJECT_ID = os.getenv('GOOGLE_CLOUD_PROJECT') TOPIC = os.getenv('PUBSUB_TOPIC') def on_pubsub_message(msg): # These cases are not handled: # - A message may be received more than once # - A message may be received out of order cmd = msg.data msg.ack() # print('.. msg: %s' % cmd) global msg_received_total global msg_received_loop global subscription if cmd == 'stop': print("Received 'stop'") msg_received_total += msg_received_loop subscription.future.set_result(True) return msg_received_loop += 1 if cmd == 'loop_end': print('Messages received: %d + %d' % ( msg_received_total, msg_received_loop)) msg_received_total += msg_received_loop msg_received_loop = 0 msg_received_total = 0 msg_received_loop = 0 subscriber = pubsub.SubscriberClient() topic_path = subscriber.topic_path(PROJECT_ID, TOPIC) subsc_path = subscriber.subscription_path(PROJECT_ID, gethostname()) try: subscriber.create_subscription(subsc_path, topic_path) except AlreadyExists: pass subscription = subscriber.subscribe(subsc_path, on_pubsub_message) print('Subscribed to messages on %s' % subsc_path) print('Waiting for future...') subscription.future.result() print('Total messages received: %d' % msg_received_total)
-
Publisher sample code
<pubsub_publish.py>import os from time import sleep from google.cloud import pubsub PROJECT_ID = os.getenv('GOOGLE_CLOUD_PROJECT') TOPIC = os.getenv('PUBSUB_TOPIC') PARAMS = [ # (number_of_messages, seconds_after_each_message) (300, 1), # 5 min (3000, 0.1), # 5 min (30000, 0.01), # 5 min (30, 10), # 5 min (5, 60), # 5 min (3, 600), # 30 min (2, 3600), # 120 min (30000, 0.01), # 5 min (3000, 0.1), # 5 min (300, 1), # 5 min ] publisher = pubsub.PublisherClient() topic_path = publisher.topic_path(PROJECT_ID, TOPIC) msg_sent = 0 for messages, delay in PARAMS: for i in xrange(1, messages): publisher.publish(topic_path, 'msg#{}'.format(i).encode('utf-8')) sleep(delay) # Keep last message to send a 'loop_end' command sleep(1) publisher.publish(topic_path, b'loop_end') sleep(1) print('Messages sent: %d + %d' % (msg_sent, messages)) msg_sent += messages # publisher.publish(topic_path, b'stop') print('Total messages sent: %d' % msg_sent)