From 79e69dea2ca8634f8aac01034aee882ee00d69b0 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Fri, 11 Aug 2023 00:10:41 +0200 Subject: [PATCH 1/3] lock ephemeral storage retrieval --- localstack/services/s3/v3/storage/ephemeral.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/localstack/services/s3/v3/storage/ephemeral.py b/localstack/services/s3/v3/storage/ephemeral.py index a7c5986e04ab3..78094b6c98c14 100644 --- a/localstack/services/s3/v3/storage/ephemeral.py +++ b/localstack/services/s3/v3/storage/ephemeral.py @@ -1,5 +1,6 @@ import base64 import hashlib +import threading from collections import defaultdict from io import BytesIO from shutil import rmtree @@ -331,6 +332,7 @@ def __init__(self, root_directory: str = None): if not root_directory: root_directory = mkdtemp() self.root_directory = root_directory + self._lock_multipart_create = threading.RLock() def open(self, bucket: BucketName, s3_object: S3Object) -> EphemeralS3StoredObject: """ @@ -403,12 +405,13 @@ def get_multipart( self, bucket: BucketName, s3_multipart: S3Multipart ) -> EphemeralS3StoredMultipart: upload_key = self._resolve_upload_directory(bucket, s3_multipart.id) - if not (multipart := self._get_multipart(bucket, upload_key)): - - upload_dir = self._create_upload_directory(bucket, s3_multipart.id) - - multipart = EphemeralS3StoredMultipart(self, bucket, s3_multipart, upload_dir) - self._filesystem[bucket]["multiparts"][upload_key] = multipart + # We need to lock this block, because we could have concurrent requests trying to access the same multipart + # and both creating it at the same time, returning 2 different entities and overriding one + with self._lock_multipart_create: + if not (multipart := self._get_multipart(bucket, upload_key)): + upload_dir = self._create_upload_directory(bucket, s3_multipart.id) + multipart = EphemeralS3StoredMultipart(self, bucket, s3_multipart, upload_dir) + self._filesystem[bucket]["multiparts"][upload_key] = multipart return multipart From 10b51d77ec3b71832e8d38a97101a8e24151f95c Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Sat, 12 Aug 2023 19:58:29 +0200 Subject: [PATCH 2/3] implement BucketLogging mocking --- localstack/services/s3/v3/models.py | 3 +- localstack/services/s3/v3/provider.py | 56 ++++++ tests/aws/s3/test_s3.py | 234 +++++++++++++------------- tests/aws/s3/test_s3.snapshot.json | 14 +- 4 files changed, 188 insertions(+), 119 deletions(-) diff --git a/localstack/services/s3/v3/models.py b/localstack/services/s3/v3/models.py index 7a254b9fee756..c021e364a41bb 100644 --- a/localstack/services/s3/v3/models.py +++ b/localstack/services/s3/v3/models.py @@ -108,7 +108,7 @@ class S3Bucket: payer: Payer encryption_rule: Optional[ServerSideEncryptionRule] public_access_block: Optional[PublicAccessBlockConfiguration] - accelerate_status: BucketAccelerateStatus + accelerate_status: Optional[BucketAccelerateStatus] object_lock_enabled: bool object_ownership: ObjectOwnership intelligent_tiering_configurations: dict[IntelligentTieringId, IntelligentTieringConfiguration] @@ -142,6 +142,7 @@ def __init__( self.public_access_block = DEFAULT_PUBLIC_BLOCK_ACCESS self.multiparts = {} self.notification_configuration = {} + self.logging = {} self.cors_rules = None self.lifecycle_rules = None self.website_configuration = None diff --git a/localstack/services/s3/v3/provider.py b/localstack/services/s3/v3/provider.py index f177e6ba297e8..72f2cbfc846ec 100644 --- a/localstack/services/s3/v3/provider.py +++ b/localstack/services/s3/v3/provider.py @@ -23,6 +23,7 @@ BucketAlreadyOwnedByYou, BucketCannedACL, BucketLifecycleConfiguration, + BucketLoggingStatus, BucketName, BucketNotEmpty, BypassGovernanceRetention, @@ -45,6 +46,7 @@ CreateBucketRequest, CreateMultipartUploadOutput, CreateMultipartUploadRequest, + CrossLocationLoggingProhibitted, Delete, DeletedObject, DeleteMarkerEntry, @@ -64,6 +66,7 @@ GetBucketInventoryConfigurationOutput, GetBucketLifecycleConfigurationOutput, GetBucketLocationOutput, + GetBucketLoggingOutput, GetBucketOwnershipControlsOutput, GetBucketPolicyOutput, GetBucketRequestPaymentOutput, @@ -96,6 +99,7 @@ InvalidPartNumber, InvalidPartOrder, InvalidStorageClass, + InvalidTargetBucketForLogging, InventoryConfiguration, InventoryId, KeyMarker, @@ -3135,6 +3139,58 @@ def put_bucket_accelerate_configuration( s3_bucket.accelerate_status = status + def put_bucket_logging( + self, + context: RequestContext, + bucket: BucketName, + bucket_logging_status: BucketLoggingStatus, + content_md5: ContentMD5 = None, + checksum_algorithm: ChecksumAlgorithm = None, + expected_bucket_owner: AccountId = None, + ) -> None: + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + + if not (logging_config := bucket_logging_status.get("LoggingEnabled")): + s3_bucket.logging = {} + return + + # the target bucket must be in the same account + if not (target_bucket_name := logging_config.get("TargetBucket")): + raise MalformedXML() + + if not logging_config.get("TargetPrefix"): + logging_config["TargetPrefix"] = "" + + # TODO: validate Grants + + if not (target_s3_bucket := store.buckets.get(bucket)): + raise InvalidTargetBucketForLogging( + "The target bucket for logging does not exist", + TargetBucket=target_bucket_name, + ) + + if target_s3_bucket.bucket_region != s3_bucket.bucket_region: + raise CrossLocationLoggingProhibitted( + "Cross S3 location logging not allowed. ", + TargetBucketLocation=target_s3_bucket.bucket_region, + ) + + s3_bucket.logging = logging_config + + def get_bucket_logging( + self, context: RequestContext, bucket: BucketName, expected_bucket_owner: AccountId = None + ) -> GetBucketLoggingOutput: + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + + if not s3_bucket.logging: + return GetBucketLoggingOutput() + + return GetBucketLoggingOutput(LoggingEnabled=s3_bucket.logging) + # ###### THIS ARE UNIMPLEMENTED METHODS TO ALLOW TESTING, DO NOT COUNT THEM AS DONE ###### # def put_bucket_acl( diff --git a/tests/aws/s3/test_s3.py b/tests/aws/s3/test_s3.py index 859f5aa223591..d38dcb1a8589e 100644 --- a/tests/aws/s3/test_s3.py +++ b/tests/aws/s3/test_s3.py @@ -5199,120 +5199,6 @@ def test_s3_get_object_headers(self, aws_client, s3_create_bucket, snapshot): aws_client.s3.get_object(Bucket=bucket, Key=key, IfMatch="etag") snapshot.match("if_match_err_1", e.value.response["Error"]) - @markers.aws.validated - def test_put_bucket_logging(self, aws_client, s3_create_bucket, snapshot): - snapshot.add_transformer( - [ - snapshot.transform.key_value("TargetBucket"), - snapshot.transform.key_value("DisplayName", reference_replacement=False), - snapshot.transform.key_value( - "ID", value_replacement="owner-id", reference_replacement=False - ), - ] - ) - - bucket_name = s3_create_bucket() - target_bucket = s3_create_bucket() - bucket_logging_status = { - "LoggingEnabled": { - "TargetBucket": target_bucket, - "TargetPrefix": "log", - }, - } - resp = aws_client.s3.get_bucket_acl(Bucket=target_bucket) - snapshot.match("get-bucket-default-acl", resp) - - # this might have been failing in the past, as the target bucket does not give access to LogDelivery to - # write/read_acp. however, AWS accepts it, because you can also set it with Permissions - resp = aws_client.s3.put_bucket_logging( - Bucket=bucket_name, BucketLoggingStatus=bucket_logging_status - ) - snapshot.match("put-bucket-logging", resp) - - resp = aws_client.s3.get_bucket_logging(Bucket=bucket_name) - snapshot.match("get-bucket-logging", resp) - - # delete BucketLogging - resp = aws_client.s3.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus={}) - snapshot.match("put-bucket-logging-delete", resp) - - @markers.aws.validated - def test_put_bucket_logging_accept_wrong_grants(self, aws_client, s3_create_bucket, snapshot): - snapshot.add_transformer(snapshot.transform.key_value("TargetBucket")) - - bucket_name = s3_create_bucket() - - target_bucket = s3_create_bucket() - # We need to delete the ObjectOwnership from the bucket, because you otherwise can't set TargetGrants on it - # TODO: have the same default as AWS and have ObjectOwnership set - aws_client.s3.delete_bucket_ownership_controls(Bucket=target_bucket) - - bucket_logging_status = { - "LoggingEnabled": { - "TargetBucket": target_bucket, - "TargetPrefix": "log", - "TargetGrants": [ - { - "Grantee": { - "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", - "Type": "Group", - }, - "Permission": "WRITE", - }, - { - "Grantee": { - "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", - "Type": "Group", - }, - "Permission": "READ_ACP", - }, - ], - }, - } - - # from the documentation, only WRITE | READ | FULL_CONTROL are allowed, but AWS let READ_ACP pass - resp = aws_client.s3.put_bucket_logging( - Bucket=bucket_name, BucketLoggingStatus=bucket_logging_status - ) - snapshot.match("put-bucket-logging", resp) - - resp = aws_client.s3.get_bucket_logging(Bucket=bucket_name) - snapshot.match("get-bucket-logging", resp) - - @markers.aws.validated - def test_put_bucket_logging_wrong_target(self, aws_client, s3_create_bucket, snapshot): - snapshot.add_transformer(snapshot.transform.key_value("TargetBucket")) - bucket_name = s3_create_bucket() - target_bucket = s3_create_bucket( - CreateBucketConfiguration={"LocationConstraint": "us-west-2"} - ) - - with pytest.raises(ClientError) as e: - bucket_logging_status = { - "LoggingEnabled": { - "TargetBucket": target_bucket, - "TargetPrefix": "log", - }, - } - aws_client.s3.put_bucket_logging( - Bucket=bucket_name, BucketLoggingStatus=bucket_logging_status - ) - snapshot.match("put-bucket-logging-different-regions", e.value.response) - - nonexistent_target_bucket = f"target-bucket-{long_uid()}" - with pytest.raises(ClientError) as e: - bucket_logging_status = { - "LoggingEnabled": { - "TargetBucket": nonexistent_target_bucket, - "TargetPrefix": "log", - }, - } - aws_client.s3.put_bucket_logging( - Bucket=bucket_name, BucketLoggingStatus=bucket_logging_status - ) - snapshot.match("put-bucket-logging-non-existent-bucket", e.value.response) - assert e.value.response["Error"]["TargetBucket"] == nonexistent_target_bucket - @markers.aws.validated def test_s3_inventory_report_crud(self, aws_client, s3_create_bucket, snapshot): snapshot.add_transformer(snapshot.transform.resource_name()) @@ -9421,6 +9307,126 @@ def test_s3_copy_object_legal_hold(self, s3_create_bucket, snapshot, aws_client) ) +class TestS3BucketLogging: + @markers.aws.validated + def test_put_bucket_logging(self, aws_client, s3_create_bucket, snapshot): + snapshot.add_transformer( + [ + snapshot.transform.key_value("TargetBucket"), + snapshot.transform.key_value("DisplayName", reference_replacement=False), + snapshot.transform.key_value( + "ID", value_replacement="owner-id", reference_replacement=False + ), + ] + ) + + bucket_name = s3_create_bucket() + target_bucket = s3_create_bucket() + + resp = aws_client.s3.get_bucket_logging(Bucket=bucket_name) + snapshot.match("get-bucket-logging-default", resp) + + bucket_logging_status = { + "LoggingEnabled": { + "TargetBucket": target_bucket, + "TargetPrefix": "log", + }, + } + resp = aws_client.s3.get_bucket_acl(Bucket=target_bucket) + snapshot.match("get-bucket-default-acl", resp) + + # this might have been failing in the past, as the target bucket does not give access to LogDelivery to + # write/read_acp. however, AWS accepts it, because you can also set it with Permissions + resp = aws_client.s3.put_bucket_logging( + Bucket=bucket_name, BucketLoggingStatus=bucket_logging_status + ) + snapshot.match("put-bucket-logging", resp) + + resp = aws_client.s3.get_bucket_logging(Bucket=bucket_name) + snapshot.match("get-bucket-logging", resp) + + # delete BucketLogging + resp = aws_client.s3.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus={}) + snapshot.match("put-bucket-logging-delete", resp) + + @markers.aws.validated + def test_put_bucket_logging_accept_wrong_grants(self, aws_client, s3_create_bucket, snapshot): + snapshot.add_transformer(snapshot.transform.key_value("TargetBucket")) + + bucket_name = s3_create_bucket() + + target_bucket = s3_create_bucket() + # We need to delete the ObjectOwnership from the bucket, because you otherwise can't set TargetGrants on it + # TODO: have the same default as AWS and have ObjectOwnership set + aws_client.s3.delete_bucket_ownership_controls(Bucket=target_bucket) + + bucket_logging_status = { + "LoggingEnabled": { + "TargetBucket": target_bucket, + "TargetPrefix": "log", + "TargetGrants": [ + { + "Grantee": { + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", + "Type": "Group", + }, + "Permission": "WRITE", + }, + { + "Grantee": { + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", + "Type": "Group", + }, + "Permission": "READ_ACP", + }, + ], + }, + } + + # from the documentation, only WRITE | READ | FULL_CONTROL are allowed, but AWS let READ_ACP pass + resp = aws_client.s3.put_bucket_logging( + Bucket=bucket_name, BucketLoggingStatus=bucket_logging_status + ) + snapshot.match("put-bucket-logging", resp) + + resp = aws_client.s3.get_bucket_logging(Bucket=bucket_name) + snapshot.match("get-bucket-logging", resp) + + @markers.aws.validated + def test_put_bucket_logging_wrong_target(self, aws_client, s3_create_bucket, snapshot): + snapshot.add_transformer(snapshot.transform.key_value("TargetBucket")) + bucket_name = s3_create_bucket() + target_bucket = s3_create_bucket( + CreateBucketConfiguration={"LocationConstraint": "us-west-2"} + ) + + with pytest.raises(ClientError) as e: + bucket_logging_status = { + "LoggingEnabled": { + "TargetBucket": target_bucket, + "TargetPrefix": "log", + }, + } + aws_client.s3.put_bucket_logging( + Bucket=bucket_name, BucketLoggingStatus=bucket_logging_status + ) + snapshot.match("put-bucket-logging-different-regions", e.value.response) + + nonexistent_target_bucket = f"target-bucket-{long_uid()}" + with pytest.raises(ClientError) as e: + bucket_logging_status = { + "LoggingEnabled": { + "TargetBucket": nonexistent_target_bucket, + "TargetPrefix": "log", + }, + } + aws_client.s3.put_bucket_logging( + Bucket=bucket_name, BucketLoggingStatus=bucket_logging_status + ) + snapshot.match("put-bucket-logging-non-existent-bucket", e.value.response) + assert e.value.response["Error"]["TargetBucket"] == nonexistent_target_bucket + + def _s3_client_custom_config(conf: Config, endpoint_url: str = None): if os.environ.get("TEST_TARGET") == "AWS_CLOUD": return boto3.client("s3", config=conf, endpoint_url=endpoint_url) diff --git a/tests/aws/s3/test_s3.snapshot.json b/tests/aws/s3/test_s3.snapshot.json index a3a1febffd347..3053fa1afc709 100644 --- a/tests/aws/s3/test_s3.snapshot.json +++ b/tests/aws/s3/test_s3.snapshot.json @@ -7573,9 +7573,15 @@ } } }, - "tests/aws/s3/test_s3.py::TestS3::test_put_bucket_logging": { - "recorded-date": "03-08-2023, 04:26:09", + "tests/aws/s3/test_s3.py::TestS3BucketLogging::test_put_bucket_logging": { + "recorded-date": "12-08-2023, 19:54:07", "recorded-content": { + "get-bucket-logging-default": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, "get-bucket-default-acl": { "Grants": [ { @@ -7620,7 +7626,7 @@ } } }, - "tests/aws/s3/test_s3.py::TestS3::test_put_bucket_logging_accept_wrong_grants": { + "tests/aws/s3/test_s3.py::TestS3BucketLogging::test_put_bucket_logging_accept_wrong_grants": { "recorded-date": "03-08-2023, 04:26:11", "recorded-content": { "put-bucket-logging": { @@ -7657,7 +7663,7 @@ } } }, - "tests/aws/s3/test_s3.py::TestS3::test_put_bucket_logging_wrong_target": { + "tests/aws/s3/test_s3.py::TestS3BucketLogging::test_put_bucket_logging_wrong_target": { "recorded-date": "03-08-2023, 04:26:14", "recorded-content": { "put-bucket-logging-different-regions": { From a3af01ce1aaf9fbbc0b212920e47290b6b1b066e Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Sat, 12 Aug 2023 20:15:25 +0200 Subject: [PATCH 3/3] implement BucketReplication mocking --- localstack/services/s3/provider.py | 8 +- localstack/services/s3/v3/models.py | 1 + localstack/services/s3/v3/provider.py | 69 +++++- tests/aws/s3/test_s3.py | 342 ++++++++++++++------------ tests/aws/s3/test_s3.snapshot.json | 28 ++- 5 files changed, 279 insertions(+), 169 deletions(-) diff --git a/localstack/services/s3/provider.py b/localstack/services/s3/provider.py index bc11e8b7df676..ceda0ebd3173f 100644 --- a/localstack/services/s3/provider.py +++ b/localstack/services/s3/provider.py @@ -933,12 +933,13 @@ def put_bucket_replication( "Versioning must be 'Enabled' on the bucket to apply a replication configuration" ) - for rule in replication_configuration.get("Rules", {}): + if not (rules := replication_configuration.get("Rules")): + raise MalformedXML() + + for rule in rules: if "ID" not in rule: rule["ID"] = short_uid() - store = self.get_store(context) - for rule in replication_configuration.get("Rules", []): dst = rule.get("Destination", {}).get("Bucket") dst_bucket_name = s3_bucket_name(dst) dst_bucket = None @@ -951,6 +952,7 @@ def put_bucket_replication( raise InvalidRequest("Destination bucket must have versioning enabled.") # TODO more validation on input + store = self.get_store(context) store.bucket_replication[bucket] = replication_configuration def get_bucket_replication( diff --git a/localstack/services/s3/v3/models.py b/localstack/services/s3/v3/models.py index c021e364a41bb..43c35fff5acfb 100644 --- a/localstack/services/s3/v3/models.py +++ b/localstack/services/s3/v3/models.py @@ -152,6 +152,7 @@ def __init__( self.analytics_configurations = {} self.inventory_configurations = {} self.object_lock_default_retention = {} + self.replication = None # see https://docs.aws.amazon.com/AmazonS3/latest/API/API_Owner.html self.owner = get_owner_for_account_id(account_id) diff --git a/localstack/services/s3/v3/provider.py b/localstack/services/s3/v3/provider.py index 72f2cbfc846ec..d17e2b3a816d2 100644 --- a/localstack/services/s3/v3/provider.py +++ b/localstack/services/s3/v3/provider.py @@ -26,6 +26,7 @@ BucketLoggingStatus, BucketName, BucketNotEmpty, + BucketVersioningStatus, BypassGovernanceRetention, ChecksumAlgorithm, ChecksumCRC32, @@ -69,6 +70,7 @@ GetBucketLoggingOutput, GetBucketOwnershipControlsOutput, GetBucketPolicyOutput, + GetBucketReplicationOutput, GetBucketRequestPaymentOutput, GetBucketTaggingOutput, GetBucketVersioningOutput, @@ -159,6 +161,8 @@ PutObjectRequest, PutObjectRetentionOutput, PutObjectTaggingOutput, + ReplicationConfiguration, + ReplicationConfigurationNotFoundError, RequestPayer, RequestPaymentConfiguration, RestoreObjectOutput, @@ -250,7 +254,8 @@ validate_website_configuration, ) from localstack.services.s3.website_hosting import register_website_hosting_routes -from localstack.utils.strings import to_str +from localstack.utils.aws.arns import s3_bucket_name +from localstack.utils.strings import short_uid, to_str LOG = logging.getLogger(__name__) @@ -3191,6 +3196,68 @@ def get_bucket_logging( return GetBucketLoggingOutput(LoggingEnabled=s3_bucket.logging) + def put_bucket_replication( + self, + context: RequestContext, + bucket: BucketName, + replication_configuration: ReplicationConfiguration, + content_md5: ContentMD5 = None, + checksum_algorithm: ChecksumAlgorithm = None, + token: ObjectLockToken = None, + expected_bucket_owner: AccountId = None, + ) -> None: + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + if not s3_bucket.versioning_status == BucketVersioningStatus.Enabled: + raise InvalidRequest( + "Versioning must be 'Enabled' on the bucket to apply a replication configuration" + ) + + if not (rules := replication_configuration.get("Rules")): + raise MalformedXML() + + for rule in rules: + if "ID" not in rule: + rule["ID"] = short_uid() + + dest_bucket_arn = rule.get("Destination", {}).get("Bucket") + dest_bucket_name = s3_bucket_name(dest_bucket_arn) + if ( + not (dest_s3_bucket := store.buckets.get(dest_bucket_name)) + or not dest_s3_bucket.versioning_status == BucketVersioningStatus.Enabled + ): + # according to AWS testing the same exception is raised if the bucket does not exist + # or if versioning was disabled + raise InvalidRequest("Destination bucket must have versioning enabled.") + + # TODO more validation on input + s3_bucket.replication = replication_configuration + + def get_bucket_replication( + self, context: RequestContext, bucket: BucketName, expected_bucket_owner: AccountId = None + ) -> GetBucketReplicationOutput: + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + + if not s3_bucket.replication: + raise ReplicationConfigurationNotFoundError( + "The replication configuration was not found", + BucketName=bucket, + ) + + return GetBucketReplicationOutput(ReplicationConfiguration=s3_bucket.replication) + + def delete_bucket_replication( + self, context: RequestContext, bucket: BucketName, expected_bucket_owner: AccountId = None + ) -> None: + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + + s3_bucket.replication = None + # ###### THIS ARE UNIMPLEMENTED METHODS TO ALLOW TESTING, DO NOT COUNT THEM AS DONE ###### # def put_bucket_acl( diff --git a/tests/aws/s3/test_s3.py b/tests/aws/s3/test_s3.py index d38dcb1a8589e..0a8b247359f3b 100644 --- a/tests/aws/s3/test_s3.py +++ b/tests/aws/s3/test_s3.py @@ -297,168 +297,6 @@ def _filter_header(param: dict) -> dict: class TestS3: - @markers.aws.validated - @pytest.mark.xfail( - condition=LEGACY_S3_PROVIDER, - reason="exceptions not raised", - ) - def test_replication_config_without_filter( - self, s3_create_bucket, create_iam_role_with_policy, snapshot, aws_client - ): - snapshot.add_transformer(snapshot.transform.s3_api()) - snapshot.add_transformer( - snapshot.transform.jsonpath( - "$..ReplicationConfiguration.Role", "role", reference_replacement=False - ) - ) - snapshot.add_transformer( - snapshot.transform.jsonpath( - "$..Destination.Bucket", "dest-bucket", reference_replacement=False - ) - ) - bucket_src = f"src-{short_uid()}" - bucket_dst = f"dst-{short_uid()}" - role_name = f"replication_role_{short_uid()}" - policy_name = f"replication_policy_{short_uid()}" - - role_arn = create_iam_role_with_policy( - RoleName=role_name, - PolicyName=policy_name, - RoleDefinition=S3_ASSUME_ROLE_POLICY, - PolicyDefinition=S3_POLICY, - ) - s3_create_bucket(Bucket=bucket_src) - # enable versioning on src - aws_client.s3.put_bucket_versioning( - Bucket=bucket_src, VersioningConfiguration={"Status": "Enabled"} - ) - - s3_create_bucket(Bucket=bucket_dst) - - replication_config = { - "Role": role_arn, - "Rules": [ - { - "ID": "rtc", - "Priority": 0, - "Filter": {}, - "Status": "Disabled", - "Destination": { - "Bucket": "arn:aws:s3:::does-not-exist", - "StorageClass": "STANDARD", - "ReplicationTime": {"Status": "Enabled", "Time": {"Minutes": 15}}, - "Metrics": {"Status": "Enabled", "EventThreshold": {"Minutes": 15}}, - }, - "DeleteMarkerReplication": {"Status": "Disabled"}, - } - ], - } - with pytest.raises(ClientError) as e: - aws_client.s3.put_bucket_replication( - ReplicationConfiguration=replication_config, Bucket=bucket_src - ) - snapshot.match("expected_error_dest_does_not_exist", e.value.response) - - # set correct destination - replication_config["Rules"][0]["Destination"]["Bucket"] = f"arn:aws:s3:::{bucket_dst}" - - with pytest.raises(ClientError) as e: - aws_client.s3.put_bucket_replication( - ReplicationConfiguration=replication_config, Bucket=bucket_src - ) - snapshot.match("expected_error_dest_versioning_disabled", e.value.response) - - # enable versioning on destination bucket - aws_client.s3.put_bucket_versioning( - Bucket=bucket_dst, VersioningConfiguration={"Status": "Enabled"} - ) - - response = aws_client.s3.put_bucket_replication( - ReplicationConfiguration=replication_config, Bucket=bucket_src - ) - snapshot.match("put-bucket-replication", response) - - response = aws_client.s3.get_bucket_replication(Bucket=bucket_src) - snapshot.match("get-bucket-replication", response) - - @markers.aws.validated - @pytest.mark.xfail( - condition=LEGACY_S3_PROVIDER, - reason="exceptions not raised", - ) - def test_replication_config( - self, s3_create_bucket, create_iam_role_with_policy, snapshot, aws_client - ): - snapshot.add_transformer(snapshot.transform.s3_api()) - snapshot.add_transformer( - snapshot.transform.jsonpath( - "$..ReplicationConfiguration.Role", "role", reference_replacement=False - ) - ) - snapshot.add_transformer( - snapshot.transform.jsonpath( - "$..Destination.Bucket", "dest-bucket", reference_replacement=False - ) - ) - snapshot.add_transformer( - snapshot.transform.key_value("ID", "id", reference_replacement=False) - ) - bucket_src = f"src-{short_uid()}" - bucket_dst = f"dst-{short_uid()}" - role_name = f"replication_role_{short_uid()}" - policy_name = f"replication_policy_{short_uid()}" - - role_arn = create_iam_role_with_policy( - RoleName=role_name, - PolicyName=policy_name, - RoleDefinition=S3_ASSUME_ROLE_POLICY, - PolicyDefinition=S3_POLICY, - ) - s3_create_bucket(Bucket=bucket_src) - - s3_create_bucket( - Bucket=bucket_dst, CreateBucketConfiguration={"LocationConstraint": "us-west-2"} - ) - aws_client.s3.put_bucket_versioning( - Bucket=bucket_dst, VersioningConfiguration={"Status": "Enabled"} - ) - - # expect error if versioning is disabled on src-bucket - with pytest.raises(ClientError) as e: - aws_client.s3.get_bucket_replication(Bucket=bucket_src) - snapshot.match("expected_error_no_replication_set", e.value.response) - - replication_config = { - "Role": role_arn, - "Rules": [ - { - "Status": "Enabled", - "Priority": 1, - "DeleteMarkerReplication": {"Status": "Disabled"}, - "Filter": {"Prefix": "Tax"}, - "Destination": {"Bucket": f"arn:aws:s3:::{bucket_dst}"}, - } - ], - } - with pytest.raises(ClientError) as e: - aws_client.s3.put_bucket_replication( - ReplicationConfiguration=replication_config, Bucket=bucket_src - ) - snapshot.match("expected_error_versioning_not_enabled", e.value.response) - - # enable versioning - aws_client.s3.put_bucket_versioning( - Bucket=bucket_src, VersioningConfiguration={"Status": "Enabled"} - ) - - response = aws_client.s3.put_bucket_replication( - ReplicationConfiguration=replication_config, Bucket=bucket_src - ) - snapshot.match("put-bucket-replication", response) - - response = aws_client.s3.get_bucket_replication(Bucket=bucket_src) - snapshot.match("get-bucket-replication", response) - @markers.aws.validated @markers.snapshot.skip_snapshot_verify( condition=is_old_provider, @@ -9427,6 +9265,186 @@ def test_put_bucket_logging_wrong_target(self, aws_client, s3_create_bucket, sna assert e.value.response["Error"]["TargetBucket"] == nonexistent_target_bucket +class TestS3BucketReplication: + @markers.aws.validated + @pytest.mark.xfail( + condition=LEGACY_S3_PROVIDER, + reason="exceptions not raised", + ) + def test_replication_config_without_filter( + self, s3_create_bucket, create_iam_role_with_policy, snapshot, aws_client + ): + snapshot.add_transformer(snapshot.transform.s3_api()) + snapshot.add_transformer( + snapshot.transform.jsonpath( + "$..ReplicationConfiguration.Role", "role", reference_replacement=False + ) + ) + snapshot.add_transformer( + snapshot.transform.jsonpath( + "$..Destination.Bucket", "dest-bucket", reference_replacement=False + ) + ) + bucket_src = f"src-{short_uid()}" + bucket_dst = f"dst-{short_uid()}" + role_name = f"replication_role_{short_uid()}" + policy_name = f"replication_policy_{short_uid()}" + + role_arn = create_iam_role_with_policy( + RoleName=role_name, + PolicyName=policy_name, + RoleDefinition=S3_ASSUME_ROLE_POLICY, + PolicyDefinition=S3_POLICY, + ) + s3_create_bucket(Bucket=bucket_src) + # enable versioning on src + aws_client.s3.put_bucket_versioning( + Bucket=bucket_src, VersioningConfiguration={"Status": "Enabled"} + ) + + s3_create_bucket(Bucket=bucket_dst) + + replication_config = { + "Role": role_arn, + "Rules": [ + { + "ID": "rtc", + "Priority": 0, + "Filter": {}, + "Status": "Disabled", + "Destination": { + "Bucket": "arn:aws:s3:::does-not-exist", + "StorageClass": "STANDARD", + "ReplicationTime": {"Status": "Enabled", "Time": {"Minutes": 15}}, + "Metrics": {"Status": "Enabled", "EventThreshold": {"Minutes": 15}}, + }, + "DeleteMarkerReplication": {"Status": "Disabled"}, + } + ], + } + with pytest.raises(ClientError) as e: + aws_client.s3.put_bucket_replication( + ReplicationConfiguration=replication_config, Bucket=bucket_src + ) + snapshot.match("expected_error_dest_does_not_exist", e.value.response) + + # set correct destination + replication_config["Rules"][0]["Destination"]["Bucket"] = f"arn:aws:s3:::{bucket_dst}" + + with pytest.raises(ClientError) as e: + aws_client.s3.put_bucket_replication( + ReplicationConfiguration=replication_config, Bucket=bucket_src + ) + snapshot.match("expected_error_dest_versioning_disabled", e.value.response) + + # enable versioning on destination bucket + aws_client.s3.put_bucket_versioning( + Bucket=bucket_dst, VersioningConfiguration={"Status": "Enabled"} + ) + + response = aws_client.s3.put_bucket_replication( + ReplicationConfiguration=replication_config, Bucket=bucket_src + ) + snapshot.match("put-bucket-replication", response) + + response = aws_client.s3.get_bucket_replication(Bucket=bucket_src) + snapshot.match("get-bucket-replication", response) + + @markers.aws.validated + @pytest.mark.xfail( + condition=LEGACY_S3_PROVIDER, + reason="exceptions not raised", + ) + def test_replication_config( + self, s3_create_bucket, create_iam_role_with_policy, snapshot, aws_client + ): + snapshot.add_transformer(snapshot.transform.s3_api()) + snapshot.add_transformer( + snapshot.transform.jsonpath( + "$..ReplicationConfiguration.Role", "role", reference_replacement=False + ) + ) + snapshot.add_transformer( + snapshot.transform.jsonpath( + "$..Destination.Bucket", "dest-bucket", reference_replacement=False + ) + ) + snapshot.add_transformer( + snapshot.transform.key_value("ID", "id", reference_replacement=False) + ) + bucket_src = f"src-{short_uid()}" + bucket_dst = f"dst-{short_uid()}" + role_name = f"replication_role_{short_uid()}" + policy_name = f"replication_policy_{short_uid()}" + + role_arn = create_iam_role_with_policy( + RoleName=role_name, + PolicyName=policy_name, + RoleDefinition=S3_ASSUME_ROLE_POLICY, + PolicyDefinition=S3_POLICY, + ) + s3_create_bucket(Bucket=bucket_src) + + s3_create_bucket( + Bucket=bucket_dst, CreateBucketConfiguration={"LocationConstraint": "us-west-2"} + ) + aws_client.s3.put_bucket_versioning( + Bucket=bucket_dst, VersioningConfiguration={"Status": "Enabled"} + ) + + # expect error if versioning is disabled on src-bucket + with pytest.raises(ClientError) as e: + aws_client.s3.get_bucket_replication(Bucket=bucket_src) + snapshot.match("expected_error_no_replication_set", e.value.response) + + replication_config = { + "Role": role_arn, + "Rules": [ + { + "Status": "Enabled", + "Priority": 1, + "DeleteMarkerReplication": {"Status": "Disabled"}, + "Filter": {"Prefix": "Tax"}, + "Destination": {"Bucket": f"arn:aws:s3:::{bucket_dst}"}, + } + ], + } + with pytest.raises(ClientError) as e: + aws_client.s3.put_bucket_replication( + ReplicationConfiguration=replication_config, Bucket=bucket_src + ) + snapshot.match("expected_error_versioning_not_enabled", e.value.response) + + # enable versioning + aws_client.s3.put_bucket_versioning( + Bucket=bucket_src, VersioningConfiguration={"Status": "Enabled"} + ) + + response = aws_client.s3.put_bucket_replication( + ReplicationConfiguration=replication_config, Bucket=bucket_src + ) + snapshot.match("put-bucket-replication", response) + + response = aws_client.s3.get_bucket_replication(Bucket=bucket_src) + snapshot.match("get-bucket-replication", response) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_bucket_replication( + Bucket=bucket_src, + ReplicationConfiguration={ + "Role": role_arn, + "Rules": [], + }, + ) + snapshot.match("put-empty-bucket-replication-rules", e.value.response) + + delete_replication = aws_client.s3.delete_bucket_replication(Bucket=bucket_src) + snapshot.match("delete-bucket-replication", delete_replication) + + delete_replication = aws_client.s3.delete_bucket_replication(Bucket=bucket_src) + snapshot.match("delete-bucket-replication-idempotent", delete_replication) + + def _s3_client_custom_config(conf: Config, endpoint_url: str = None): if os.environ.get("TEST_TARGET") == "AWS_CLOUD": return boto3.client("s3", config=conf, endpoint_url=endpoint_url) diff --git a/tests/aws/s3/test_s3.snapshot.json b/tests/aws/s3/test_s3.snapshot.json index 3053fa1afc709..84611678bf4f5 100644 --- a/tests/aws/s3/test_s3.snapshot.json +++ b/tests/aws/s3/test_s3.snapshot.json @@ -4098,8 +4098,8 @@ } } }, - "tests/aws/s3/test_s3.py::TestS3::test_replication_config": { - "recorded-date": "03-08-2023, 04:13:08", + "tests/aws/s3/test_s3.py::TestS3BucketReplication::test_replication_config": { + "recorded-date": "12-08-2023, 20:11:46", "recorded-content": { "expected_error_no_replication_set": { "Error": { @@ -4152,6 +4152,28 @@ "HTTPHeaders": {}, "HTTPStatusCode": 200 } + }, + "put-empty-bucket-replication-rules": { + "Error": { + "Code": "MalformedXML", + "Message": "The XML you provided was not well-formed or did not validate against our published schema" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "delete-bucket-replication": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 204 + } + }, + "delete-bucket-replication-idempotent": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 204 + } } } }, @@ -4166,7 +4188,7 @@ } } }, - "tests/aws/s3/test_s3.py::TestS3::test_replication_config_without_filter": { + "tests/aws/s3/test_s3.py::TestS3BucketReplication::test_replication_config_without_filter": { "recorded-date": "03-08-2023, 04:13:02", "recorded-content": { "expected_error_dest_does_not_exist": {