Thanks to visit codestin.com
Credit goes to github.com

Skip to content

refactor SNS async publishing & ASF data models #7267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jan 3, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions localstack/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -725,6 +725,9 @@ def in_docker():
"ES_MULTI_CLUSTER"
)

# Whether to really publish to GCM while using SNS Platform Application (needs credentials)
LEGACY_SNS_GCM_PUBLISHING = is_env_true("LEGACY_SNS_GCM_PUBLISHING")

# TODO remove fallback to LAMBDA_DOCKER_NETWORK with next minor version
MAIN_DOCKER_NETWORK = os.environ.get("MAIN_DOCKER_NETWORK", "") or LAMBDA_DOCKER_NETWORK

Expand Down Expand Up @@ -795,6 +798,7 @@ def in_docker():
"LEGACY_DIRECTORIES",
"LEGACY_DOCKER_CLIENT",
"LEGACY_EDGE_PROXY",
"LEGACY_SNS_GCM_PUBLISHING",
"LOCALSTACK_API_KEY",
"LOCALSTACK_HOSTNAME",
"LOG_LICENSE_ISSUES",
Expand Down
33 changes: 33 additions & 0 deletions localstack/services/sns/constants.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import re
from string import ascii_letters, digits

SNS_PROTOCOLS = [
"http",
"https",
"email",
"email-json",
"sms",
"sqs",
"application",
"lambda",
"firehose",
]

VALID_SUBSCRIPTION_ATTR_NAME = [
"DeliveryPolicy",
"FilterPolicy",
"FilterPolicyScope",
"RawMessageDelivery",
"RedrivePolicy",
"SubscriptionRoleArn",
]

MSG_ATTR_NAME_REGEX = re.compile(r"^(?!\.)(?!.*\.$)(?!.*\.\.)[a-zA-Z0-9_\-.]+$")
ATTR_TYPE_REGEX = re.compile(r"^(String|Number|Binary)\..+$")
VALID_MSG_ATTR_NAME_CHARS = set(ascii_letters + digits + "." + "-" + "_")


GCM_URL = "https://fcm.googleapis.com/fcm/send"

# Endpoint to access all the PlatformEndpoint sent Messages
PLATFORM_ENDPOINT_MSGS_ENDPOINT = "/_aws/sns/platform-endpoint-messages"
91 changes: 88 additions & 3 deletions localstack/services/sns/models.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,93 @@
from typing import Dict, List
from dataclasses import dataclass, field
from typing import Dict, List, Literal, Optional, TypedDict, Union

from localstack.aws.api.sns import (
MessageAttributeMap,
PublishBatchRequestEntry,
subscriptionARN,
topicARN,
)
from localstack.services.stores import AccountRegionBundle, BaseStore, LocalAttribute
from localstack.utils.strings import long_uid

SnsProtocols = Literal[
"http", "https", "email", "email-json", "sms", "sqs", "application", "lambda", "firehose"
]

SnsApplicationPlatforms = Literal[
"APNS", "APNS_SANDBOX", "ADM", "FCM", "Baidu", "GCM", "MPNS", "WNS"
]

SnsMessageProtocols = Literal[SnsProtocols, SnsApplicationPlatforms]


@dataclass
class SnsMessage:
type: str
message: Union[
str, Dict
] # can be Dict if after being JSON decoded for validation if structure is `json`
message_attributes: Optional[MessageAttributeMap] = None
message_structure: Optional[str] = None
subject: Optional[str] = None
message_deduplication_id: Optional[str] = None
message_group_id: Optional[str] = None
token: Optional[str] = None
message_id: str = field(default_factory=long_uid)

def __post_init__(self):
if self.message_attributes is None:
self.message_attributes = {}

def message_content(self, protocol: SnsMessageProtocols) -> str:
"""
Helper function to retrieve the message content for the right protocol if the StructureMessage is `json`
See https://docs.aws.amazon.com/sns/latest/dg/sns-send-custom-platform-specific-payloads-mobile-devices.html
https://docs.aws.amazon.com/sns/latest/dg/example_sns_Publish_section.html
:param protocol:
:return: message content as string
"""
if self.message_structure == "json":
return self.message.get(protocol, self.message.get("default"))

return self.message

@classmethod
def from_batch_entry(cls, entry: PublishBatchRequestEntry) -> "SnsMessage":
return cls(
type="Notification",
message=entry["Message"],
subject=entry.get("Subject"),
message_structure=entry.get("MessageStructure"),
message_attributes=entry.get("MessageAttributes"),
message_deduplication_id=entry.get("MessageDeduplicationId"),
message_group_id=entry.get("MessageGroupId"),
)


class SnsSubscription(TypedDict):
"""
In SNS, Subscription can be represented with only TopicArn, Endpoint, Protocol, SubscriptionArn and Owner, for
example in ListSubscriptions. However, when getting a subscription with GetSubscriptionAttributes, it will return
the Subscription object merged with its own attributes.
This represents this merged object, for internal use and in GetSubscriptionAttributes
https://docs.aws.amazon.com/cli/latest/reference/sns/get-subscription-attributes.html
"""

TopicArn: topicARN
Endpoint: str
Protocol: SnsProtocols
SubscriptionArn: subscriptionARN
PendingConfirmation: Literal["true", "false"]
Owner: Optional[str]
FilterPolicy: Optional[str]
FilterPolicyScope: Literal["MessageAttributes", "MessageBody"]
RawMessageDelivery: Literal["true", "false"]


class SnsStore(BaseStore):
# maps topic ARN to list of subscriptions
sns_subscriptions: Dict[str, List[Dict]] = LocalAttribute(default=dict)
# maps topic ARN to topic's subscriptions
sns_subscriptions: Dict[str, List[SnsSubscription]] = LocalAttribute(default=dict)

# maps subscription ARN to subscription status
subscription_status: Dict[str, Dict] = LocalAttribute(default=dict)
Expand All @@ -19,5 +101,8 @@ class SnsStore(BaseStore):
# list of sent SMS messages - TODO: expose via internal API
sms_messages: List[Dict] = LocalAttribute(default=list)

# filter policy are stored as JSON string in subscriptions, store the decoded result Dict
subscription_filter_policy: Dict[subscriptionARN, Dict] = LocalAttribute(default=dict)


sns_stores = AccountRegionBundle("sns", SnsStore)
Loading