From c5be4d85b349e33e6f8e0bdba95709c4db3f94db Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Mon, 13 Feb 2023 13:53:53 +0100 Subject: [PATCH 1/2] fix modifying batch context for all subscribers --- localstack/services/sns/provider.py | 2 + localstack/services/sns/publisher.py | 18 ++- tests/integration/test_sns.py | 111 ++++++++++++++++ tests/integration/test_sns.snapshot.json | 160 +++++++++++++++++++++++ 4 files changed, 286 insertions(+), 5 deletions(-) diff --git a/localstack/services/sns/provider.py b/localstack/services/sns/provider.py index daa0dbf8b8f97..6791662cec0cb 100644 --- a/localstack/services/sns/provider.py +++ b/localstack/services/sns/provider.py @@ -495,6 +495,8 @@ def get_subscription_attributes( removed_attrs = ["sqs_queue_url"] if "FilterPolicyScope" in sub and "FilterPolicy" not in sub: removed_attrs.append("FilterPolicyScope") + elif "FilterPolicy" in sub and "FilterPolicyScope" not in sub: + sub["FilterPolicyScope"] = "MessageAttributes" attributes = {k: v for k, v in sub.items() if k not in removed_attrs} return GetSubscriptionAttributesResponse(Attributes=attributes) diff --git a/localstack/services/sns/publisher.py b/localstack/services/sns/publisher.py index 597ed3a377e53..76cf8d60efb26 100644 --- a/localstack/services/sns/publisher.py +++ b/localstack/services/sns/publisher.py @@ -1,6 +1,7 @@ import abc import ast import base64 +import copy import datetime import hashlib import json @@ -1101,21 +1102,22 @@ def publish_batch_to_topic(self, ctx: SnsBatchPublishContext, topic_arn: str) -> notifier = self.batch_topic_notifiers.get(protocol) # does the notifier supports batching natively? for now, only SQS supports it if notifier: + subscriber_ctx = ctx messages_amount_before_filtering = len(ctx.messages) - ctx.messages = [ + filtered_messages = [ message for message in ctx.messages if self._should_publish(ctx.store, message, subscriber) ] - if not ctx.messages: + if not filtered_messages: LOG.debug( "No messages match filter policy, not publishing batch from topic '%s' to subscription '%s'", topic_arn, subscriber["SubscriptionArn"], ) - return + continue - messages_amount = len(ctx.messages) + messages_amount = len(filtered_messages) if messages_amount != messages_amount_before_filtering: LOG.debug( "After applying subscription filter, %s out of %s message(s) to be sent to '%s'", @@ -1123,6 +1125,10 @@ def publish_batch_to_topic(self, ctx: SnsBatchPublishContext, topic_arn: str) -> messages_amount_before_filtering, subscriber["SubscriptionArn"], ) + # We need to copy the context to not overwrite the messages after filtering messages, otherwise we + # would filter on the same context for different subscribers + subscriber_ctx = copy.copy(ctx) + subscriber_ctx.messages = filtered_messages LOG.debug( "Topic '%s' batch publishing %s messages to subscribed '%s' with protocol '%s' (subscription '%s')", @@ -1132,7 +1138,9 @@ def publish_batch_to_topic(self, ctx: SnsBatchPublishContext, topic_arn: str) -> subscriber["Protocol"], subscriber["SubscriptionArn"], ) - self.executor.submit(notifier.publish, context=ctx, subscriber=subscriber) + self.executor.submit( + notifier.publish, context=subscriber_ctx, subscriber=subscriber + ) else: # if no batch support, fall back to sending them sequentially notifier = self.topic_notifiers[subscriber["Protocol"]] diff --git a/tests/integration/test_sns.py b/tests/integration/test_sns.py index 39d22de75502c..93e4646dd4678 100644 --- a/tests/integration/test_sns.py +++ b/tests/integration/test_sns.py @@ -3263,3 +3263,114 @@ def test_publish_to_fifo_with_target_arn(self, sns_client, sns_create_topic): MessageGroupId="123", ) assert "MessageId" in response + + @pytest.mark.aws_validated + @pytest.mark.skip_snapshot_verify(paths=["$..Attributes.SubscriptionPrincipal"]) + def test_filter_policy_for_batch( + self, + sns_client, + sqs_client, + sqs_create_queue, + sns_create_topic, + sns_create_sqs_subscription, + snapshot, + ): + + topic_arn = sns_create_topic()["TopicArn"] + queue_url_with_filter = sqs_create_queue() + subscription_with_filter = sns_create_sqs_subscription( + topic_arn=topic_arn, queue_url=queue_url_with_filter + ) + subscription_with_filter_arn = subscription_with_filter["SubscriptionArn"] + + queue_url_no_filter = sqs_create_queue() + subscription_no_filter = sns_create_sqs_subscription( + topic_arn=topic_arn, queue_url=queue_url_no_filter + ) + subscription_no_filter_arn = subscription_no_filter["SubscriptionArn"] + + filter_policy = {"attr1": [{"numeric": [">", 0, "<=", 100]}]} + sns_client.set_subscription_attributes( + SubscriptionArn=subscription_with_filter_arn, + AttributeName="FilterPolicy", + AttributeValue=json.dumps(filter_policy), + ) + + response_attributes = sns_client.get_subscription_attributes( + SubscriptionArn=subscription_with_filter_arn + ) + snapshot.match("subscription-attributes-with-filter", response_attributes) + + response_attributes = sns_client.get_subscription_attributes( + SubscriptionArn=subscription_no_filter_arn + ) + snapshot.match("subscription-attributes-no-filter", response_attributes) + + sqs_wait_time = 4 if is_aws_cloud() else 1 + + response_before_publish_no_filter = sqs_client.receive_message( + QueueUrl=queue_url_with_filter, VisibilityTimeout=0, WaitTimeSeconds=sqs_wait_time + ) + snapshot.match("messages-no-filter-before-publish", response_before_publish_no_filter) + + response_before_publish_filter = sqs_client.receive_message( + QueueUrl=queue_url_with_filter, VisibilityTimeout=0, WaitTimeSeconds=sqs_wait_time + ) + snapshot.match("messages-with-filter-before-publish", response_before_publish_filter) + + # publish message that satisfies the filter policy, assert that message is received + message = "This is a test message" + message_attributes = {"attr1": {"DataType": "Number", "StringValue": "99"}} + sns_client.publish_batch( + TopicArn=topic_arn, + PublishBatchRequestEntries=[ + { + "Id": "1", + "Message": message, + "MessageAttributes": message_attributes, + } + ], + ) + + response_after_publish_no_filter = sqs_client.receive_message( + QueueUrl=queue_url_no_filter, VisibilityTimeout=0, WaitTimeSeconds=sqs_wait_time + ) + snapshot.match("messages-no-filter-after-publish-ok", response_after_publish_no_filter) + sqs_client.delete_message( + QueueUrl=queue_url_no_filter, + ReceiptHandle=response_after_publish_no_filter["Messages"][0]["ReceiptHandle"], + ) + + response_after_publish_filter = sqs_client.receive_message( + QueueUrl=queue_url_with_filter, VisibilityTimeout=0, WaitTimeSeconds=sqs_wait_time + ) + snapshot.match("messages-with-filter-after-publish-ok", response_after_publish_filter) + sqs_client.delete_message( + QueueUrl=queue_url_with_filter, + ReceiptHandle=response_after_publish_filter["Messages"][0]["ReceiptHandle"], + ) + + # publish message that does not satisfy the filter policy, assert that message is not received by the + # subscription with the filter and received by the other + sns_client.publish_batch( + TopicArn=topic_arn, + PublishBatchRequestEntries=[ + { + "Id": "1", + "Message": "This is another test message", + "MessageAttributes": {"attr1": {"DataType": "Number", "StringValue": "111"}}, + } + ], + ) + + response_after_publish_no_filter = sqs_client.receive_message( + QueueUrl=queue_url_no_filter, VisibilityTimeout=0, WaitTimeSeconds=sqs_wait_time + ) + # there should be 1 message in the queue, latest sent + snapshot.match("messages-no-filter-after-publish-ok-1", response_after_publish_no_filter) + + response_after_publish_filter = sqs_client.receive_message( + QueueUrl=queue_url_with_filter, VisibilityTimeout=0, WaitTimeSeconds=sqs_wait_time + ) + # there should be no messages in this queue + snapshot.match("messages-with-filter-after-publish-filtered", response_after_publish_filter) diff --git a/tests/integration/test_sns.snapshot.json b/tests/integration/test_sns.snapshot.json index 43c1db2a26938..6f71cda976512 100644 --- a/tests/integration/test_sns.snapshot.json +++ b/tests/integration/test_sns.snapshot.json @@ -3040,5 +3040,165 @@ } } } + }, + "tests/integration/test_sns.py::TestSNSProvider::test_filter_policy_for_batch": { + "recorded-date": "13-02-2023, 13:40:06", + "recorded-content": { + "subscription-attributes-with-filter": { + "Attributes": { + "ConfirmationWasAuthenticated": "true", + "Endpoint": "arn:aws:sqs::111111111111:", + "FilterPolicy": { + "attr1": [ + { + "numeric": [ + ">", + 0, + "<=", + 100 + ] + } + ] + }, + "FilterPolicyScope": "MessageAttributes", + "Owner": "111111111111", + "PendingConfirmation": "false", + "Protocol": "sqs", + "RawMessageDelivery": "false", + "SubscriptionArn": "arn:aws:sns::111111111111::", + "SubscriptionPrincipal": "", + "TopicArn": "arn:aws:sns::111111111111:" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "subscription-attributes-no-filter": { + "Attributes": { + "ConfirmationWasAuthenticated": "true", + "Endpoint": "arn:aws:sqs::111111111111:", + "Owner": "111111111111", + "PendingConfirmation": "false", + "Protocol": "sqs", + "RawMessageDelivery": "false", + "SubscriptionArn": "arn:aws:sns::111111111111::", + "SubscriptionPrincipal": "", + "TopicArn": "arn:aws:sns::111111111111:" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "messages-no-filter-before-publish": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "messages-with-filter-before-publish": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "messages-no-filter-after-publish-ok": { + "Messages": [ + { + "Body": { + "Type": "Notification", + "MessageId": "", + "TopicArn": "arn:aws:sns::111111111111:", + "Message": "This is a test message", + "Timestamp": "date", + "SignatureVersion": "1", + "Signature": "", + "SigningCertURL": "https://sns..amazonaws.com/SimpleNotificationService-", + "UnsubscribeURL": "/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns::111111111111::", + "MessageAttributes": { + "attr1": { + "Type": "Number", + "Value": "99" + } + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "messages-with-filter-after-publish-ok": { + "Messages": [ + { + "Body": { + "Type": "Notification", + "MessageId": "", + "TopicArn": "arn:aws:sns::111111111111:", + "Message": "This is a test message", + "Timestamp": "date", + "SignatureVersion": "1", + "Signature": "", + "SigningCertURL": "https://sns..amazonaws.com/SimpleNotificationService-", + "UnsubscribeURL": "/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns::111111111111::", + "MessageAttributes": { + "attr1": { + "Type": "Number", + "Value": "99" + } + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "messages-no-filter-after-publish-ok-1": { + "Messages": [ + { + "Body": { + "Type": "Notification", + "MessageId": "", + "TopicArn": "arn:aws:sns::111111111111:", + "Message": "This is another test message", + "Timestamp": "date", + "SignatureVersion": "1", + "Signature": "", + "SigningCertURL": "https://sns..amazonaws.com/SimpleNotificationService-", + "UnsubscribeURL": "/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns::111111111111::", + "MessageAttributes": { + "attr1": { + "Type": "Number", + "Value": "111" + } + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "messages-with-filter-after-publish-filtered": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } } } From d2bb5371bd668a971384821451d681f505e8ca6a Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Mon, 13 Feb 2023 16:50:54 +0100 Subject: [PATCH 2/2] regenerate snapshots for sub attributes --- tests/integration/test_sns.py | 3 +++ tests/integration/test_sns.snapshot.json | 14 +++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_sns.py b/tests/integration/test_sns.py index 93e4646dd4678..c7cad78bebfa2 100644 --- a/tests/integration/test_sns.py +++ b/tests/integration/test_sns.py @@ -241,6 +241,7 @@ def test_attribute_raw_subscribe( snapshot.match("messages-response", response) @pytest.mark.aws_validated + @pytest.mark.skip_snapshot_verify(paths=["$..Attributes.SubscriptionPrincipal"]) def test_filter_policy( self, sns_client, @@ -308,6 +309,7 @@ def test_filter_policy( assert num_msgs_2 == num_msgs_1 @pytest.mark.aws_validated + @pytest.mark.skip_snapshot_verify(paths=["$..Attributes.SubscriptionPrincipal"]) def test_exists_filter_policy( self, sns_client, @@ -432,6 +434,7 @@ def get_filter_policy(): assert num_msgs_4 == num_msgs_3 @pytest.mark.aws_validated + @pytest.mark.skip_snapshot_verify(paths=["$..Attributes.SubscriptionPrincipal"]) def test_subscribe_sqs_queue( self, sns_client, diff --git a/tests/integration/test_sns.snapshot.json b/tests/integration/test_sns.snapshot.json index 6f71cda976512..e9fab53976c18 100644 --- a/tests/integration/test_sns.snapshot.json +++ b/tests/integration/test_sns.snapshot.json @@ -242,7 +242,7 @@ } }, "tests/integration/test_sns.py::TestSNSProvider::test_filter_policy": { - "recorded-date": "09-08-2022, 11:30:09", + "recorded-date": "13-02-2023, 16:48:57", "recorded-content": { "subscription-attributes": { "Attributes": { @@ -260,11 +260,13 @@ } ] }, + "FilterPolicyScope": "MessageAttributes", "Owner": "111111111111", "PendingConfirmation": "false", "Protocol": "sqs", "RawMessageDelivery": "false", "SubscriptionArn": "arn:aws:sns::111111111111::", + "SubscriptionPrincipal": "", "TopicArn": "arn:aws:sns::111111111111:" }, "ResponseMetadata": { @@ -341,7 +343,7 @@ } }, "tests/integration/test_sns.py::TestSNSProvider::test_exists_filter_policy": { - "recorded-date": "09-08-2022, 11:30:12", + "recorded-date": "13-02-2023, 16:49:47", "recorded-content": { "subscription-attributes-policy-1": { "Attributes": { @@ -354,11 +356,13 @@ } ] }, + "FilterPolicyScope": "MessageAttributes", "Owner": "111111111111", "PendingConfirmation": "false", "Protocol": "sqs", "RawMessageDelivery": "false", "SubscriptionArn": "arn:aws:sns::111111111111::", + "SubscriptionPrincipal": "", "TopicArn": "arn:aws:sns::111111111111:" }, "ResponseMetadata": { @@ -451,11 +455,13 @@ } ] }, + "FilterPolicyScope": "MessageAttributes", "Owner": "111111111111", "PendingConfirmation": "false", "Protocol": "sqs", "RawMessageDelivery": "false", "SubscriptionArn": "arn:aws:sns::111111111111::", + "SubscriptionPrincipal": "", "TopicArn": "arn:aws:sns::111111111111:" }, "ResponseMetadata": { @@ -526,7 +532,7 @@ } }, "tests/integration/test_sns.py::TestSNSProvider::test_subscribe_sqs_queue": { - "recorded-date": "09-08-2022, 11:30:15", + "recorded-date": "13-02-2023, 16:50:28", "recorded-content": { "subscription-attributes": { "Attributes": { @@ -544,11 +550,13 @@ } ] }, + "FilterPolicyScope": "MessageAttributes", "Owner": "111111111111", "PendingConfirmation": "false", "Protocol": "sqs", "RawMessageDelivery": "false", "SubscriptionArn": "arn:aws:sns::111111111111::", + "SubscriptionPrincipal": "", "TopicArn": "arn:aws:sns::111111111111:" }, "ResponseMetadata": {