From 494d194aafaf5e0ddf6b60325d002b73fdcc404f Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Wed, 6 Feb 2019 16:41:47 -0800 Subject: [PATCH 1/4] Quickstart V2 --- pubsub/cloud-client/quickstart/pub.py | 71 ++++++++++++++++ pubsub/cloud-client/quickstart/pub_test.py | 69 +++++++++++++++ pubsub/cloud-client/quickstart/sub.py | 62 ++++++++++++++ pubsub/cloud-client/quickstart/sub_test.py | 98 ++++++++++++++++++++++ 4 files changed, 300 insertions(+) create mode 100644 pubsub/cloud-client/quickstart/pub.py create mode 100644 pubsub/cloud-client/quickstart/pub_test.py create mode 100644 pubsub/cloud-client/quickstart/sub.py create mode 100644 pubsub/cloud-client/quickstart/sub_test.py diff --git a/pubsub/cloud-client/quickstart/pub.py b/pubsub/cloud-client/quickstart/pub.py new file mode 100644 index 00000000000..81ff50113ad --- /dev/null +++ b/pubsub/cloud-client/quickstart/pub.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START pubsub_quickstart_pub_all] +import argparse +import time +# [START pubsub_quickstart_pub_deps] +from google.cloud import pubsub_v1 +# [END pubsub_quickstart_pub_deps] + + +def get_callback(api_future, data): + """Wraps message data in the context of the callback function.""" + def callback(api_future): + if api_future.exception(): + print("There was a problem with message {}".format(data)) + else: + print("Published message {} now has message ID {}".format( + data, api_future.result())) + return callback + + +def pub(project_id, topic_name): + """Publishes a message to a Pub/Sub topic.""" + # [START pubsub_quickstart_pub_client] + # Initializes the Publisher client + client = pubsub_v1.PublisherClient() + # [END pubsub_quickstart_pub_client] + # Creates a fully qualified identifier in the form of + # `projects/{project_id}/topics/{topic_name}` + topic_path = client.topic_path(project_id, topic_name) + + # Data sent to Cloud Pub/Sub must be a bytestring + data = "Hello, World!" + data = data.encode('utf-8') + + # When you publish a message, the client returns a future. + api_future = client.publish(topic_path, data=data) + api_future.add_done_callback(get_callback(api_future, data)) + + # Keeps the main thread from exiting to handle message processing + # in the background. + while True: + time.sleep(60) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Google Cloud project ID') + parser.add_argument('topic_name', help='Pub/Sub topic name') + + args = parser.parse_args() + + pub(args.project_id, args.topic_name) +# [END pubsub_quickstart_pub_all] diff --git a/pubsub/cloud-client/quickstart/pub_test.py b/pubsub/cloud-client/quickstart/pub_test.py new file mode 100644 index 00000000000..eb7131b2d2d --- /dev/null +++ b/pubsub/cloud-client/quickstart/pub_test.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import os +import pytest +import time + +from google.cloud import pubsub_v1 + +import pub + +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'quickstart-pub-test-topic' + + +@pytest.fixture(scope='module') +def publisher_client(): + yield pubsub_v1.PublisherClient() + + +@pytest.fixture(scope='module') +def topic(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, TOPIC) + + try: + publisher_client.delete_topic(topic_path) + except Exception: + pass + + publisher_client.create_topic(topic_path) + + yield TOPIC + + +def _make_sleep_patch(): + real_sleep = time.sleep + + def new_sleep(period): + if period == 60: + real_sleep(10) + raise RuntimeError('sigil') + else: + real_sleep(period) + + return mock.patch('time.sleep', new=new_sleep) + + +def test_pub(topic, capsys): + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + pub.pub(PROJECT, topic) + + out, _ = capsys.readouterr() + + assert "Published message b'Hello, World!'" in out diff --git a/pubsub/cloud-client/quickstart/sub.py b/pubsub/cloud-client/quickstart/sub.py new file mode 100644 index 00000000000..dec10076bfb --- /dev/null +++ b/pubsub/cloud-client/quickstart/sub.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START pubsub_quickstart_sub_all] +import argparse +import time +# [START pubsub_quickstart_sub_deps] +from google.cloud import pubsub_v1 +# [END pubsub_quickstart_sub_deps] + + +def sub(project_id, subscription_name): + """Receives messages from a Pub/Sub subscription.""" + # [START pubsub_quickstart_sub_client] + # Initializes the Subscriber client + client = pubsub_v1.SubscriberClient() + # [END pubsub_quickstart_sub_client] + # Creates a fully qualified identifier in the form of + # `projects/{project_id}/subscriptions/{subscription_name}` + subscription_path = client.subscription_path( + project_id, subscription_name) + + def callback(message): + print('Received message {} of message ID {}'.format( + message, message.message_id)) + # Acknowledges the message. Unack'ed messages will be redelivered. + message.ack() + + client.subscribe(subscription_path, callback=callback) + print('Listening for messages on {}'.format(subscription_path)) + + # The subscriber is non-blocking. We must keep the main thread from + # exiting so it can process messages asynchronously in the background. + while True: + time.sleep(60) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Google Cloud project ID') + parser.add_argument('subscription_name', help='Pub/Sub subscription name') + + args = parser.parse_args() + + sub(args.project_id, args.subscription_name) +# [END pubsub_quickstart_sub_all] diff --git a/pubsub/cloud-client/quickstart/sub_test.py b/pubsub/cloud-client/quickstart/sub_test.py new file mode 100644 index 00000000000..f907210f1bb --- /dev/null +++ b/pubsub/cloud-client/quickstart/sub_test.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import os +import pytest +import time + +from google.cloud import pubsub_v1 + +import sub + + +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'quickstart-sub-test-topic' +SUBSCRIPTION = 'quickstart-sub-test-topic-sub' + + +@pytest.fixture(scope='module') +def publisher_client(): + yield pubsub_v1.PublisherClient() + + +@pytest.fixture(scope='module') +def topic(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, TOPIC) + + try: + publisher_client.delete_topic(topic_path) + except Exception: + pass + + publisher_client.create_topic(topic_path) + + yield topic_path + + +@pytest.fixture(scope='module') +def subscriber_client(): + yield pubsub_v1.SubscriberClient() + + +@pytest.fixture(scope='module') +def subscription(subscriber_client, topic): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + + try: + subscriber_client.delete_subscription(subscription_path) + except Exception: + pass + + subscriber_client.create_subscription(subscription_path, topic=topic) + + yield SUBSCRIPTION + + +def _publish_messages(publisher_client, topic): + data = u'Hello, World!'.encode('utf-8') + publisher_client.publish(topic, data=data) + + +def _make_sleep_patch(): + real_sleep = time.sleep + + def new_sleep(period): + if period == 60: + real_sleep(10) + raise RuntimeError('sigil') + else: + real_sleep(period) + + return mock.patch('time.sleep', new=new_sleep) + + +def test_sub(publisher_client, topic, subscription, capsys): + _publish_messages(publisher_client, topic) + + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + sub.sub(PROJECT, subscription) + + out, _ = capsys.readouterr() + + assert "Received message" in out From c3768c4ee9b78b7b5a3d8f7814ec73883fcb2c0b Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Thu, 7 Feb 2019 11:31:38 -0800 Subject: [PATCH 2/4] Adopts Kir's suggestions --- pubsub/cloud-client/quickstart/pub.py | 13 ++++++++----- pubsub/cloud-client/quickstart/pub_test.py | 19 +------------------ pubsub/cloud-client/quickstart/sub.py | 8 +++++--- 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/pubsub/cloud-client/quickstart/pub.py b/pubsub/cloud-client/quickstart/pub.py index 81ff50113ad..f8154b36107 100644 --- a/pubsub/cloud-client/quickstart/pub.py +++ b/pubsub/cloud-client/quickstart/pub.py @@ -24,12 +24,15 @@ def get_callback(api_future, data): """Wraps message data in the context of the callback function.""" + def callback(api_future): - if api_future.exception(): - print("There was a problem with message {}".format(data)) - else: + try: print("Published message {} now has message ID {}".format( data, api_future.result())) + except Exception: + print("A problem occurred when publishing {}: {}\n".format( + data, api_future.exception())) + raise return callback @@ -53,8 +56,8 @@ def pub(project_id, topic_name): # Keeps the main thread from exiting to handle message processing # in the background. - while True: - time.sleep(60) + while api_future.running(): + time.sleep(0.1) if __name__ == '__main__': diff --git a/pubsub/cloud-client/quickstart/pub_test.py b/pubsub/cloud-client/quickstart/pub_test.py index eb7131b2d2d..39c86bf0c19 100644 --- a/pubsub/cloud-client/quickstart/pub_test.py +++ b/pubsub/cloud-client/quickstart/pub_test.py @@ -14,10 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock import os import pytest -import time from google.cloud import pubsub_v1 @@ -46,23 +44,8 @@ def topic(publisher_client): yield TOPIC -def _make_sleep_patch(): - real_sleep = time.sleep - - def new_sleep(period): - if period == 60: - real_sleep(10) - raise RuntimeError('sigil') - else: - real_sleep(period) - - return mock.patch('time.sleep', new=new_sleep) - - def test_pub(topic, capsys): - with _make_sleep_patch(): - with pytest.raises(RuntimeError, match='sigil'): - pub.pub(PROJECT, topic) + pub.pub(PROJECT, topic) out, _ = capsys.readouterr() diff --git a/pubsub/cloud-client/quickstart/sub.py b/pubsub/cloud-client/quickstart/sub.py index dec10076bfb..30263196c88 100644 --- a/pubsub/cloud-client/quickstart/sub.py +++ b/pubsub/cloud-client/quickstart/sub.py @@ -38,12 +38,14 @@ def callback(message): message, message.message_id)) # Acknowledges the message. Unack'ed messages will be redelivered. message.ack() + print('Acknolwedged message of message ID {}\n'.format( + message.message_id)) client.subscribe(subscription_path, callback=callback) - print('Listening for messages on {}'.format(subscription_path)) + print('Listening for messages on {}..\n'.format(subscription_path)) - # The subscriber is non-blocking. We must keep the main thread from - # exiting so it can process messages asynchronously in the background. + # The subscriber is non-blocking. We keep the main thread from exiting + # so it can process messages asynchronously in the background. while True: time.sleep(60) From a1c303b1a8f77cb9a944674e01d83a96b086e7cf Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 8 Feb 2019 15:55:53 -0800 Subject: [PATCH 3/4] Adopted Tim's suggestions --- pubsub/cloud-client/quickstart/pub.py | 13 ++++---- pubsub/cloud-client/quickstart/pub_test.py | 15 ++++++--- pubsub/cloud-client/quickstart/sub.py | 12 +++---- pubsub/cloud-client/quickstart/sub_test.py | 38 ++++++++++++---------- 4 files changed, 43 insertions(+), 35 deletions(-) diff --git a/pubsub/cloud-client/quickstart/pub.py b/pubsub/cloud-client/quickstart/pub.py index f8154b36107..9617b34ea84 100644 --- a/pubsub/cloud-client/quickstart/pub.py +++ b/pubsub/cloud-client/quickstart/pub.py @@ -23,7 +23,7 @@ def get_callback(api_future, data): - """Wraps message data in the context of the callback function.""" + """Wrap message data in the context of the callback function.""" def callback(api_future): try: @@ -39,23 +39,22 @@ def callback(api_future): def pub(project_id, topic_name): """Publishes a message to a Pub/Sub topic.""" # [START pubsub_quickstart_pub_client] - # Initializes the Publisher client + # Initialize a Publisher client client = pubsub_v1.PublisherClient() # [END pubsub_quickstart_pub_client] - # Creates a fully qualified identifier in the form of + # Create a fully qualified identifier in the form of # `projects/{project_id}/topics/{topic_name}` topic_path = client.topic_path(project_id, topic_name) # Data sent to Cloud Pub/Sub must be a bytestring - data = "Hello, World!" - data = data.encode('utf-8') + data = b"Hello, World!" # When you publish a message, the client returns a future. api_future = client.publish(topic_path, data=data) api_future.add_done_callback(get_callback(api_future, data)) - # Keeps the main thread from exiting to handle message processing - # in the background. + # Keep the main thread from exiting until background message + # is processed. while api_future.running(): time.sleep(0.1) diff --git a/pubsub/cloud-client/quickstart/pub_test.py b/pubsub/cloud-client/quickstart/pub_test.py index 39c86bf0c19..b2ba84f8d51 100644 --- a/pubsub/cloud-client/quickstart/pub_test.py +++ b/pubsub/cloud-client/quickstart/pub_test.py @@ -17,6 +17,7 @@ import os import pytest +from google.api_core.exceptions import AlreadyExists from google.cloud import pubsub_v1 import pub @@ -35,18 +36,22 @@ def topic(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) try: - publisher_client.delete_topic(topic_path) - except Exception: + publisher_client.create_topic(topic_path) + except AlreadyExists: pass - publisher_client.create_topic(topic_path) - yield TOPIC -def test_pub(topic, capsys): +def test_pub(publisher_client, topic, capsys): pub.pub(PROJECT, topic) out, _ = capsys.readouterr() assert "Published message b'Hello, World!'" in out + + # Clean up. + publisher_client.delete_topic('projects/{}/topics/{}'.format( + PROJECT, + TOPIC + )) diff --git a/pubsub/cloud-client/quickstart/sub.py b/pubsub/cloud-client/quickstart/sub.py index 30263196c88..520803d70a5 100644 --- a/pubsub/cloud-client/quickstart/sub.py +++ b/pubsub/cloud-client/quickstart/sub.py @@ -25,10 +25,10 @@ def sub(project_id, subscription_name): """Receives messages from a Pub/Sub subscription.""" # [START pubsub_quickstart_sub_client] - # Initializes the Subscriber client + # Initialize a Subscriber client client = pubsub_v1.SubscriberClient() # [END pubsub_quickstart_sub_client] - # Creates a fully qualified identifier in the form of + # Create a fully qualified identifier in the form of # `projects/{project_id}/subscriptions/{subscription_name}` subscription_path = client.subscription_path( project_id, subscription_name) @@ -36,16 +36,16 @@ def sub(project_id, subscription_name): def callback(message): print('Received message {} of message ID {}'.format( message, message.message_id)) - # Acknowledges the message. Unack'ed messages will be redelivered. + # Acknowledge the message. Unack'ed messages will be redelivered. message.ack() - print('Acknolwedged message of message ID {}\n'.format( + print('Acknowledged message of message ID {}\n'.format( message.message_id)) client.subscribe(subscription_path, callback=callback) print('Listening for messages on {}..\n'.format(subscription_path)) - # The subscriber is non-blocking. We keep the main thread from exiting - # so it can process messages asynchronously in the background. + # Keep the main thread from exiting so the subscriber can + # process messages in the background. while True: time.sleep(60) diff --git a/pubsub/cloud-client/quickstart/sub_test.py b/pubsub/cloud-client/quickstart/sub_test.py index f907210f1bb..88e6b6da188 100644 --- a/pubsub/cloud-client/quickstart/sub_test.py +++ b/pubsub/cloud-client/quickstart/sub_test.py @@ -19,6 +19,7 @@ import pytest import time +from google.api_core.exceptions import AlreadyExists from google.cloud import pubsub_v1 import sub @@ -35,16 +36,14 @@ def publisher_client(): @pytest.fixture(scope='module') -def topic(publisher_client): +def topic_path(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) try: - publisher_client.delete_topic(topic_path) - except Exception: + publisher_client.create_topic(topic_path) + except AlreadyExists: pass - publisher_client.create_topic(topic_path) - yield topic_path @@ -54,25 +53,18 @@ def subscriber_client(): @pytest.fixture(scope='module') -def subscription(subscriber_client, topic): +def subscription(subscriber_client, topic_path): subscription_path = subscriber_client.subscription_path( PROJECT, SUBSCRIPTION) try: - subscriber_client.delete_subscription(subscription_path) - except Exception: + subscriber_client.create_subscription(subscription_path, topic_path) + except AlreadyExists: pass - subscriber_client.create_subscription(subscription_path, topic=topic) - yield SUBSCRIPTION -def _publish_messages(publisher_client, topic): - data = u'Hello, World!'.encode('utf-8') - publisher_client.publish(topic, data=data) - - def _make_sleep_patch(): real_sleep = time.sleep @@ -86,8 +78,13 @@ def new_sleep(period): return mock.patch('time.sleep', new=new_sleep) -def test_sub(publisher_client, topic, subscription, capsys): - _publish_messages(publisher_client, topic) +def test_sub(publisher_client, + topic_path, + subscriber_client, + subscription, + capsys): + + publisher_client.publish(topic_path, data=b'Hello, World!') with _make_sleep_patch(): with pytest.raises(RuntimeError, match='sigil'): @@ -96,3 +93,10 @@ def test_sub(publisher_client, topic, subscription, capsys): out, _ = capsys.readouterr() assert "Received message" in out + assert "Acknowledged message" in out + + # Clean up. + subscriber_client.delete_subscription( + 'projects/{}/subscriptions/{}'.format(PROJECT, SUBSCRIPTION) + ) + publisher_client.delete_topic(topic_path) From cb03cd1f0c4026621acd89ffb15113ee5464e62a Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 11 Feb 2019 15:38:08 -0800 Subject: [PATCH 4/4] proper resource deletion during teardown --- pubsub/cloud-client/quickstart/pub_test.py | 18 ++++++++++------ pubsub/cloud-client/quickstart/sub_test.py | 25 ++++++++++++++++------ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/pubsub/cloud-client/quickstart/pub_test.py b/pubsub/cloud-client/quickstart/pub_test.py index b2ba84f8d51..09443364a3f 100644 --- a/pubsub/cloud-client/quickstart/pub_test.py +++ b/pubsub/cloud-client/quickstart/pub_test.py @@ -43,15 +43,19 @@ def topic(publisher_client): yield TOPIC -def test_pub(publisher_client, topic, capsys): +@pytest.fixture +def to_delete(publisher_client): + doomed = [] + yield doomed + for item in doomed: + publisher_client.delete_topic(item) + + +def test_pub(publisher_client, topic, to_delete, capsys): pub.pub(PROJECT, topic) + to_delete.append('projects/{}/topics/{}'.format(PROJECT, TOPIC)) + out, _ = capsys.readouterr() assert "Published message b'Hello, World!'" in out - - # Clean up. - publisher_client.delete_topic('projects/{}/topics/{}'.format( - PROJECT, - TOPIC - )) diff --git a/pubsub/cloud-client/quickstart/sub_test.py b/pubsub/cloud-client/quickstart/sub_test.py index 88e6b6da188..9c70384ed69 100644 --- a/pubsub/cloud-client/quickstart/sub_test.py +++ b/pubsub/cloud-client/quickstart/sub_test.py @@ -65,6 +65,17 @@ def subscription(subscriber_client, topic_path): yield SUBSCRIPTION +@pytest.fixture +def to_delete(publisher_client, subscriber_client): + doomed = [] + yield doomed + for client, item in doomed: + if 'topics' in item: + publisher_client.delete_topic(item) + if 'subscriptions' in item: + subscriber_client.delete_subscription(item) + + def _make_sleep_patch(): real_sleep = time.sleep @@ -82,21 +93,21 @@ def test_sub(publisher_client, topic_path, subscriber_client, subscription, + to_delete, capsys): publisher_client.publish(topic_path, data=b'Hello, World!') + to_delete.append((publisher_client, topic_path)) + with _make_sleep_patch(): with pytest.raises(RuntimeError, match='sigil'): sub.sub(PROJECT, subscription) - out, _ = capsys.readouterr() + to_delete.append((subscriber_client, + 'projects/{}/subscriptions/{}'.format(PROJECT, + SUBSCRIPTION))) + out, _ = capsys.readouterr() assert "Received message" in out assert "Acknowledged message" in out - - # Clean up. - subscriber_client.delete_subscription( - 'projects/{}/subscriptions/{}'.format(PROJECT, SUBSCRIPTION) - ) - publisher_client.delete_topic(topic_path)