Thanks to visit codestin.com
Credit goes to github.com

Skip to content

SNS time of message publication optimized #3105

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Oct 16, 2020
Merged

SNS time of message publication optimized #3105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0a2c3fb
unused line deleted
pinzon Oct 13, 2020
0c51210
publish_message function optimized to work with threads
pinzon Oct 14, 2020
7dace0a
sns publications are async now, test should wait
pinzon Oct 16, 2020
1edcf63
hope for publication to end
pinzon Oct 16, 2020
a7f24d7
time.sleep changed for retries
pinzon Oct 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion localstack/services/kinesis/kinesis_listener.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,6 @@ def return_response(self, method, path, data, headers, response):
return response

def decode_content(self, data):
# return json.loads(to_str(data))
try:
return json.loads(to_str(data))
except UnicodeDecodeError:
Expand Down
27 changes: 15 additions & 12 deletions localstack/services/sns/sns_listener.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from localstack.utils.aws import aws_stack
from localstack.utils.aws.aws_responses import response_regex_replace
from localstack.utils.aws.dead_letter_queue import sns_error_to_dead_letter_queue
from localstack.utils.common import timestamp_millis, short_uid, to_str, to_bytes
from localstack.utils.common import timestamp_millis, short_uid, to_str, to_bytes, start_thread
from localstack.utils.persistence import PersistingProxyListener

# set up logger
Expand Down Expand Up @@ -257,17 +257,7 @@ def unsubscribe_sqs_queue(queue_url):
subscriptions.remove(subscriber)


def publish_message(topic_arn, req_data, subscription_arn=None, skip_checks=False):
message = req_data['Message'][0]
message_id = str(uuid.uuid4())

LOG.debug('Publishing message to TopicArn: %s | Message: %s' % (topic_arn, message))

if topic_arn and ':endpoint/' in topic_arn:
# cache messages published to platform endpoints
cache = PLATFORM_ENDPOINT_MESSAGES[topic_arn] = PLATFORM_ENDPOINT_MESSAGES.get(topic_arn) or []
cache.append(req_data)

def message_to_subscribers(message_id, message, topic_arn, req_data, subscription_arn=None, skip_checks=False):
subscriptions = SNS_SUBSCRIPTIONS.get(topic_arn, [])
for subscriber in list(subscriptions):
if subscription_arn not in [None, subscriber['SubscriptionArn']]:
Expand Down Expand Up @@ -375,6 +365,19 @@ def publish_message(topic_arn, req_data, subscription_arn=None, skip_checks=Fals
else:
LOG.warning('Unexpected protocol "%s" for SNS subscription' % subscriber['Protocol'])


def publish_message(topic_arn, req_data, subscription_arn=None, skip_checks=False):
message = req_data['Message'][0]
message_id = str(uuid.uuid4())

if topic_arn and ':endpoint/' in topic_arn:
# cache messages published to platform endpoints
cache = PLATFORM_ENDPOINT_MESSAGES[topic_arn] = PLATFORM_ENDPOINT_MESSAGES.get(topic_arn) or []
cache.append(req_data)

LOG.debug('Publishing message to TopicArn: %s | Message: %s' % (topic_arn, message))
start_thread(
lambda _: message_to_subscribers(message_id, message, topic_arn, req_data, subscription_arn, skip_checks))
return message_id


Expand Down
51 changes: 28 additions & 23 deletions tests/integration/test_notifications.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@
from io import BytesIO
from localstack.utils import testutil
from localstack.utils.aws import aws_stack
from localstack.utils.common import to_str, short_uid
from localstack.utils.common import to_str, short_uid, retry

TEST_BUCKET_NAME_WITH_NOTIFICATIONS = 'test-bucket-notif-1'
TEST_QUEUE_NAME_FOR_S3 = 'test_queue'
TEST_TOPIC_NAME = 'test_topic_name_for_sqs'
TEST_S3_TOPIC_NAME = 'test_topic_name_for_s3_to_sns_to_sqs'
TEST_QUEUE_NAME_FOR_SNS = 'test_queue_for_sns'
PUBLICATION_TIMEOUT = 0.500
PUBLICATION_RETRIES = 4


class TestNotifications(unittest.TestCase):
Expand All @@ -36,14 +38,16 @@ def test_sns_to_sqs(self):
sns_client.publish(TopicArn=topic_info['TopicArn'], Message='test message for SQS',
MessageAttributes={'attr1': {'DataType': 'String', 'StringValue': test_value}})

# receive, assert, and delete message from SQS
queue_url = queue_info['QueueUrl']
assertions = []
# make sure we receive the correct topic ARN in notifications
assertions.append({'TopicArn': topic_info['TopicArn']})
# make sure the notification contains message attributes
assertions.append({'Value': test_value})
self._receive_assert_delete(queue_url, assertions, sqs_client)
def assert_message():
# receive, assert, and delete message from SQS
queue_url = queue_info['QueueUrl']
assertions = []
# make sure we receive the correct topic ARN in notifications
assertions.append({'TopicArn': topic_info['TopicArn']})
# make sure the notification contains message attributes
assertions.append({'Value': test_value})
self._receive_assert_delete(queue_url, assertions, sqs_client)
retry(assert_message, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)

def test_bucket_notifications(self):

Expand Down Expand Up @@ -191,20 +195,21 @@ def test_bucket_notifications(self):
s3_client.upload_fileobj(BytesIO(test_data2), TEST_BUCKET_NAME_WITH_NOTIFICATIONS, test_key2)

# verify subject and records

response = sqs_client.receive_message(QueueUrl=queue_url)
for message in response['Messages']:
snsObj = json.loads(message['Body'])
testutil.assert_object({'Subject': 'Amazon S3 Notification'}, snsObj)
notificationObj = json.loads(snsObj['Message'])
testutil.assert_objects(
[
{'key': test_key2},
{'name': TEST_BUCKET_NAME_WITH_NOTIFICATIONS}
], notificationObj['Records'])

sqs_client.delete_message(QueueUrl=queue_url, ReceiptHandle=message['ReceiptHandle'])

def verify():
response = sqs_client.receive_message(QueueUrl=queue_url)
for message in response['Messages']:
snsObj = json.loads(message['Body'])
testutil.assert_object({'Subject': 'Amazon S3 Notification'}, snsObj)
notificationObj = json.loads(snsObj['Message'])
testutil.assert_objects(
[
{'key': test_key2},
{'name': TEST_BUCKET_NAME_WITH_NOTIFICATIONS}
], notificationObj['Records'])

sqs_client.delete_message(QueueUrl=queue_url, ReceiptHandle=message['ReceiptHandle'])

retry(verify, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)
self._delete_notification_config()

def _delete_notification_config(self):
Expand Down
109 changes: 75 additions & 34 deletions tests/integration/test_sns.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
TEST_QUEUE_DLQ_NAME = 'TestQueue_DLQ_snsTest'
TEST_TOPIC_NAME_2 = 'topic-test-2'

PUBLICATION_TIMEOUT = .500
PUBLICATION_RETRIES = 4

THIS_FOLDER = os.path.dirname(os.path.realpath(__file__))
TEST_LAMBDA_ECHO_FILE = os.path.join(THIS_FOLDER, 'lambdas', 'lambda_echo.py')

Expand Down Expand Up @@ -54,11 +57,14 @@ def test_publish_unicode_chars(self):
# publish message to SNS, receive it from SQS, assert that messages are equal
message = u'ö§a1"_!?,. £$-'
self.sns_client.publish(TopicArn=self.topic_arn, Message=message)
msgs = self.sqs_client.receive_message(QueueUrl=queue_url)
msg_received = msgs['Messages'][0]
msg_received = json.loads(to_str(msg_received['Body']))
msg_received = msg_received['Message']
self.assertEqual(message, msg_received)

def check_message():
msgs = self.sqs_client.receive_message(QueueUrl=queue_url)
msg_received = msgs['Messages'][0]
msg_received = json.loads(to_str(msg_received['Body']))
msg_received = msg_received['Message']
self.assertEqual(message, msg_received)
retry(check_message, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)

# clean up
self.sqs_client.delete_queue(QueueUrl=queue_url)
Expand Down Expand Up @@ -124,11 +130,14 @@ def test_attribute_raw_subscribe(self):
self.sns_client.publish(TopicArn=self.topic_arn, Message=message,
MessageAttributes={'store': {'DataType': 'Binary', 'BinaryValue': binary_attribute}})

msgs = self.sqs_client.receive_message(QueueUrl=self.queue_url, MessageAttributeNames=['All'])
msg_received = msgs['Messages'][0]
def check_message():
msgs = self.sqs_client.receive_message(QueueUrl=self.queue_url, MessageAttributeNames=['All'])
msg_received = msgs['Messages'][0]

self.assertEqual(message, msg_received['Body'])
self.assertEqual(binary_attribute, msg_received['MessageAttributes']['store']['BinaryValue'])

self.assertEqual(message, msg_received['Body'])
self.assertEqual(binary_attribute, msg_received['MessageAttributes']['store']['BinaryValue'])
retry(check_message, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)

def test_filter_policy(self):
# connect SNS topic to an SQS queue
Expand All @@ -151,15 +160,23 @@ def test_filter_policy(self):
message = u'This is a test message'
self.sns_client.publish(TopicArn=self.topic_arn, Message=message,
MessageAttributes={'attr1': {'DataType': 'Number', 'StringValue': '99'}})
num_msgs_1 = len(self.sqs_client.receive_message(QueueUrl=queue_url, VisibilityTimeout=0)['Messages'])
self.assertEqual(num_msgs_1, num_msgs_0 + 1)

def check_message():
num_msgs_1 = len(self.sqs_client.receive_message(QueueUrl=queue_url, VisibilityTimeout=0)['Messages'])
self.assertEqual(num_msgs_1, num_msgs_0 + 1)
return num_msgs_1
num_msgs_1 = retry(check_message, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)

# publish message that does not satisfy the filter policy, assert that message is not received
message = u'This is a test message'
self.sns_client.publish(TopicArn=self.topic_arn, Message=message,
MessageAttributes={'attr1': {'DataType': 'Number', 'StringValue': '111'}})
num_msgs_2 = len(self.sqs_client.receive_message(QueueUrl=queue_url, VisibilityTimeout=0)['Messages'])
self.assertEqual(num_msgs_2, num_msgs_1)

def check_message2():
num_msgs_2 = len(self.sqs_client.receive_message(QueueUrl=queue_url, VisibilityTimeout=0)['Messages'])
self.assertEqual(num_msgs_2, num_msgs_1)
return num_msgs_2
retry(check_message2, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)

# clean up
self.sqs_client.delete_queue(QueueUrl=queue_url)
Expand Down Expand Up @@ -188,15 +205,23 @@ def do_subscribe(self, filter_policy, queue_arn):
self.sns_client.publish(TopicArn=self.topic_arn, Message=message,
MessageAttributes={'store': {'DataType': 'Number', 'StringValue': '99'},
'def': {'DataType': 'Number', 'StringValue': '99'}})
num_msgs_1 = len(self.sqs_client.receive_message(QueueUrl=queue_url, VisibilityTimeout=0)['Messages'])
self.assertEqual(num_msgs_1, num_msgs_0 + 1)

def check_message1():
num_msgs_1 = len(self.sqs_client.receive_message(QueueUrl=queue_url, VisibilityTimeout=0)['Messages'])
self.assertEqual(num_msgs_1, num_msgs_0 + 1)
return num_msgs_1
num_msgs_1 = retry(check_message1, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)

# publish message that does not satisfy the filter policy, assert that message is not received
message = u'This is a test message'
self.sns_client.publish(TopicArn=self.topic_arn, Message=message,
MessageAttributes={'attr1': {'DataType': 'Number', 'StringValue': '111'}})
num_msgs_2 = len(self.sqs_client.receive_message(QueueUrl=queue_url, VisibilityTimeout=0)['Messages'])
self.assertEqual(num_msgs_2, num_msgs_1)

def check_message2():
num_msgs_2 = len(self.sqs_client.receive_message(QueueUrl=queue_url, VisibilityTimeout=0)['Messages'])
self.assertEqual(num_msgs_2, num_msgs_1)
return num_msgs_2
retry(check_message2, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)

# test with exist operator set to false.
queue_arn = aws_stack.sqs_queue_arn(TEST_QUEUE_NAME)
Expand All @@ -210,17 +235,27 @@ def do_subscribe(self, filter_policy, queue_arn):
self.sns_client.publish(TopicArn=self.topic_arn, Message=message,
MessageAttributes={'store': {'DataType': 'Number', 'StringValue': '99'},
'def': {'DataType': 'Number', 'StringValue': '99'}})
num_msgs_1 = len(self.sqs_client.receive_message(QueueUrl=self.queue_url,

def check_message():
num_msgs_1 = len(self.sqs_client.receive_message(QueueUrl=self.queue_url,
VisibilityTimeout=0).get('Messages', []))
self.assertEqual(num_msgs_1, num_msgs_0)
self.assertEqual(num_msgs_1, num_msgs_0)
return num_msgs_1

num_msgs_1 = retry(check_message, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)

# publish message that without the attribute and see if its getting filtered.
message = u'This is a test message'
self.sns_client.publish(TopicArn=self.topic_arn, Message=message,
MessageAttributes={'attr1': {'DataType': 'Number', 'StringValue': '111'}})
num_msgs_2 = len(self.sqs_client.receive_message(QueueUrl=self.queue_url,
VisibilityTimeout=0).get('Messages', []))
self.assertEqual(num_msgs_2, num_msgs_1)

def check_message3():
num_msgs_2 = len(self.sqs_client.receive_message(QueueUrl=self.queue_url,
VisibilityTimeout=0).get('Messages', []))
self.assertEqual(num_msgs_2, num_msgs_1)
return num_msgs_2

retry(check_message3, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)

# clean up
self.sqs_client.delete_queue(QueueUrl=queue_url)
Expand All @@ -232,8 +267,10 @@ def test_subscribe_sqs_queue(self):
subscription = self._publish_sns_message_with_attrs(queue_arn, 'sqs')

# assert that message is received
messages = self.sqs_client.receive_message(QueueUrl=queue_url, VisibilityTimeout=0)['Messages']
self.assertEqual(json.loads(messages[0]['Body'])['MessageAttributes']['attr1']['Value'], '99.12')
def check_message():
messages = self.sqs_client.receive_message(QueueUrl=queue_url, VisibilityTimeout=0)['Messages']
self.assertEqual(json.loads(messages[0]['Body'])['MessageAttributes']['attr1']['Value'], '99.12')
retry(check_message, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)

# clean up
self.sqs_client.delete_queue(QueueUrl=queue_url)
Expand All @@ -248,7 +285,7 @@ def test_subscribe_platform_endpoint(self):
# assert that message has been received
def check_message():
self.assertGreater(len(sns_listener.PLATFORM_ENDPOINT_MESSAGES[platform_arn]), 0)
retry(check_message)
retry(check_message, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)

# clean up
sns.unsubscribe(SubscriptionArn=subscription['SubscriptionArn'])
Expand All @@ -270,6 +307,7 @@ def _publish_sns_message_with_attrs(self, endpoint_arn, protocol):
message = u'This is a test message'
self.sns_client.publish(TopicArn=self.topic_arn, Message=message,
MessageAttributes={'attr1': {'DataType': 'Number', 'StringValue': '99.12'}})
time.sleep(PUBLICATION_TIMEOUT)
return subscription

def test_unknown_topic_publish(self):
Expand Down Expand Up @@ -355,16 +393,19 @@ def test_topic_subscription(self):
Protocol='email',
Endpoint='[email protected]'
)
subscription_arn = subscription['SubscriptionArn']
subscription_obj = sns_listener.SUBSCRIPTION_STATUS[subscription_arn]
self.assertEqual(subscription_obj['Status'], 'Not Subscribed')

_token = subscription_obj['Token']
self.sns_client.confirm_subscription(
TopicArn=self.topic_arn,
Token=_token
)
self.assertEqual(subscription_obj['Status'], 'Subscribed')
def check_subscription():
subscription_arn = subscription['SubscriptionArn']
subscription_obj = sns_listener.SUBSCRIPTION_STATUS[subscription_arn]
self.assertEqual(subscription_obj['Status'], 'Not Subscribed')

_token = subscription_obj['Token']
self.sns_client.confirm_subscription(
TopicArn=self.topic_arn,
Token=_token
)
self.assertEqual(subscription_obj['Status'], 'Subscribed')
retry(check_subscription, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)

def test_dead_letter_queue(self):
lambda_name = 'test-%s' % short_uid()
Expand Down