The public access block configuration was not found
", "exception": true } + }, + { + "op": "add", + "path": "/shapes/NoSuchBucketPolicy", + "value": { + "type": "structure", + "members": { + "BucketName": { + "shape": "BucketName" + } + }, + "error": { + "httpStatusCode": 404 + }, + "documentation": "The bucket policy does not exist
", + "exception": true + } } ] } diff --git a/localstack/services/s3/exceptions.py b/localstack/services/s3/exceptions.py index 0ea863a1c7274..da76db00f82b3 100644 --- a/localstack/services/s3/exceptions.py +++ b/localstack/services/s3/exceptions.py @@ -41,3 +41,8 @@ def __init__(self, message=None): class NoSuchObjectLockConfiguration(CommonServiceException): def __init__(self, message=None): super().__init__("NoSuchObjectLockConfiguration", status_code=404, message=message) + + +class MalformedPolicy(CommonServiceException): + def __init__(self, message=None): + super().__init__("MalformedPolicy", status_code=400, message=message) diff --git a/localstack/services/s3/v3/models.py b/localstack/services/s3/v3/models.py index 7a55970f27f1d..7a254b9fee756 100644 --- a/localstack/services/s3/v3/models.py +++ b/localstack/services/s3/v3/models.py @@ -145,6 +145,8 @@ def __init__( self.cors_rules = None self.lifecycle_rules = None self.website_configuration = None + self.policy = None + self.accelerate_status = None self.intelligent_tiering_configurations = {} self.analytics_configurations = {} self.inventory_configurations = {} diff --git a/localstack/services/s3/v3/provider.py b/localstack/services/s3/v3/provider.py index cd149b86c4f25..f177e6ba297e8 100644 --- a/localstack/services/s3/v3/provider.py +++ b/localstack/services/s3/v3/provider.py @@ -1,6 +1,7 @@ import base64 import copy import datetime +import json import logging from collections import defaultdict from operator import itemgetter @@ -11,6 +12,7 @@ from localstack.aws.api.s3 import ( MFA, AbortMultipartUploadOutput, + AccelerateConfiguration, AccessControlPolicy, AccessDenied, AccountId, @@ -32,6 +34,7 @@ CommonPrefix, CompletedMultipartUpload, CompleteMultipartUploadOutput, + ConfirmRemoveSelfBucketAccess, ContentMD5, CopyObjectOutput, CopyObjectRequest, @@ -53,6 +56,7 @@ Error, Expiration, FetchOwner, + GetBucketAccelerateConfigurationOutput, GetBucketAnalyticsConfigurationOutput, GetBucketCorsOutput, GetBucketEncryptionOutput, @@ -61,6 +65,7 @@ GetBucketLifecycleConfigurationOutput, GetBucketLocationOutput, GetBucketOwnershipControlsOutput, + GetBucketPolicyOutput, GetBucketRequestPaymentOutput, GetBucketTaggingOutput, GetBucketVersioningOutput, @@ -111,6 +116,7 @@ MultipartUpload, MultipartUploadId, NoSuchBucket, + NoSuchBucketPolicy, NoSuchCORSConfiguration, NoSuchKey, NoSuchLifecycleConfiguration, @@ -139,6 +145,7 @@ Part, PartNumber, PartNumberMarker, + Policy, PreconditionFailed, Prefix, PublicAccessBlockConfiguration, @@ -186,6 +193,7 @@ InvalidBucketState, InvalidLocationConstraint, InvalidRequest, + MalformedPolicy, MalformedXML, NoSuchConfiguration, NoSuchObjectLockConfiguration, @@ -3035,6 +3043,98 @@ def delete_public_access_block( s3_bucket.public_access_block = None + def get_bucket_policy( + self, context: RequestContext, bucket: BucketName, expected_bucket_owner: AccountId = None + ) -> GetBucketPolicyOutput: + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + if not s3_bucket.policy: + raise NoSuchBucketPolicy( + "The bucket policy does not exist", + BucketName=bucket, + ) + return GetBucketPolicyOutput(Policy=s3_bucket.policy) + + def put_bucket_policy( + self, + context: RequestContext, + bucket: BucketName, + policy: Policy, + content_md5: ContentMD5 = None, + checksum_algorithm: ChecksumAlgorithm = None, + confirm_remove_self_bucket_access: ConfirmRemoveSelfBucketAccess = None, + expected_bucket_owner: AccountId = None, + ) -> None: + # TODO: there is not validation of the policy at the moment, as there was none in moto + # we store the JSON policy as is, as we do not need to decode it + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + + if not policy or policy[0] != "{": + raise MalformedPolicy("Policies must be valid JSON and the first byte must be '{'") + try: + json_policy = json.loads(policy) + if not json_policy: + # TODO: add more validation around the policy? + raise MalformedPolicy("Missing required field Statement") + except ValueError: + raise MalformedPolicy("Policies must be valid JSON and the first byte must be '{'") + + s3_bucket.policy = policy + + def delete_bucket_policy( + self, context: RequestContext, bucket: BucketName, expected_bucket_owner: AccountId = None + ) -> None: + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + + s3_bucket.policy = None + + def get_bucket_accelerate_configuration( + self, + context: RequestContext, + bucket: BucketName, + expected_bucket_owner: AccountId = None, + request_payer: RequestPayer = None, + ) -> GetBucketAccelerateConfigurationOutput: + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + + response = GetBucketAccelerateConfigurationOutput() + if s3_bucket.accelerate_status: + response["Status"] = s3_bucket.accelerate_status + + return response + + def put_bucket_accelerate_configuration( + self, + context: RequestContext, + bucket: BucketName, + accelerate_configuration: AccelerateConfiguration, + expected_bucket_owner: AccountId = None, + checksum_algorithm: ChecksumAlgorithm = None, + ) -> None: + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + + if "." in bucket: + raise InvalidRequest( + "S3 Transfer Acceleration is not supported for buckets with periods (.) in their names" + ) + + if not (status := accelerate_configuration.get("Status")) or status not in ( + "Enabled", + "Suspended", + ): + raise MalformedXML() + + s3_bucket.accelerate_status = status + # ###### THIS ARE UNIMPLEMENTED METHODS TO ALLOW TESTING, DO NOT COUNT THEM AS DONE ###### # def put_bucket_acl( diff --git a/tests/aws/s3/test_s3_api.py b/tests/aws/s3/test_s3_api.py index 300942f0d8d77..ee53dcabba6fe 100644 --- a/tests/aws/s3/test_s3_api.py +++ b/tests/aws/s3/test_s3_api.py @@ -1,3 +1,4 @@ +import json from operator import itemgetter import pytest @@ -1314,3 +1315,122 @@ def test_crud_public_access_block(self, s3_bucket, aws_client, snapshot): delete_public_access_block = aws_client.s3.delete_public_access_block(Bucket=s3_bucket) snapshot.match("idempotent-delete-public-access-block", delete_public_access_block) + + +@pytest.mark.skipif( + condition=not config.NATIVE_S3_PROVIDER, + reason="These are WIP tests for the new native S3 provider", +) +class TestS3BucketPolicy: + @markers.aws.validated + def test_bucket_policy_crud(self, s3_bucket, snapshot, aws_client): + snapshot.add_transformer(snapshot.transform.key_value("Resource")) + snapshot.add_transformer(snapshot.transform.key_value("BucketName")) + # delete the OwnershipControls so that we can set a Policy + aws_client.s3.delete_bucket_ownership_controls(Bucket=s3_bucket) + aws_client.s3.delete_public_access_block(Bucket=s3_bucket) + + # get the default Policy, should raise + with pytest.raises(ClientError) as e: + aws_client.s3.get_bucket_policy(Bucket=s3_bucket) + snapshot.match("get-bucket-default-policy", e.value.response) + + # put bucket policy + policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "s3:GetObject", + "Effect": "Allow", + "Resource": f"arn:aws:s3:::{s3_bucket}/*", + "Principal": {"AWS": "*"}, + } + ], + } + response = aws_client.s3.put_bucket_policy(Bucket=s3_bucket, Policy=json.dumps(policy)) + snapshot.match("put-bucket-policy", response) + + # retrieve and check policy config + response = aws_client.s3.get_bucket_policy(Bucket=s3_bucket) + snapshot.match("get-bucket-policy", response) + assert policy == json.loads(response["Policy"]) + + response = aws_client.s3.delete_bucket_policy(Bucket=s3_bucket) + snapshot.match("delete-bucket-policy", response) + + with pytest.raises(ClientError) as e: + aws_client.s3.get_bucket_policy(Bucket=s3_bucket) + snapshot.match("get-bucket-policy-after-delete", e.value.response) + + @markers.aws.validated + def test_bucket_policy_exc(self, s3_bucket, snapshot, aws_client): + # delete the OwnershipControls so that we can set a Policy + aws_client.s3.delete_bucket_ownership_controls(Bucket=s3_bucket) + aws_client.s3.delete_public_access_block(Bucket=s3_bucket) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_bucket_policy(Bucket=s3_bucket, Policy="") + snapshot.match("put-empty-bucket-policy", e.value.response) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_bucket_policy(Bucket=s3_bucket, Policy="invalid json") + snapshot.match("put-bucket-policy-randomstring", e.value.response) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_bucket_policy(Bucket=s3_bucket, Policy="{}") + snapshot.match("put-bucket-policy-empty-json", e.value.response) + + +@pytest.mark.skipif( + condition=not config.NATIVE_S3_PROVIDER, + reason="These are WIP tests for the new native S3 provider", +) +class TestS3BucketAccelerateConfiguration: + @markers.aws.validated + def test_bucket_acceleration_configuration_crud(self, s3_bucket, snapshot, aws_client): + get_default_config = aws_client.s3.get_bucket_accelerate_configuration(Bucket=s3_bucket) + snapshot.match("get-bucket-default-accelerate-config", get_default_config) + + response = aws_client.s3.put_bucket_accelerate_configuration( + Bucket=s3_bucket, + AccelerateConfiguration={"Status": "Enabled"}, + ) + snapshot.match("put-bucket-accelerate-config-enabled", response) + + response = aws_client.s3.get_bucket_accelerate_configuration(Bucket=s3_bucket) + snapshot.match("get-bucket-accelerate-config-enabled", response) + + response = aws_client.s3.put_bucket_accelerate_configuration( + Bucket=s3_bucket, + AccelerateConfiguration={"Status": "Suspended"}, + ) + snapshot.match("put-bucket-accelerate-config-disabled", response) + + response = aws_client.s3.get_bucket_accelerate_configuration(Bucket=s3_bucket) + snapshot.match("get-bucket-accelerate-config-disabled", response) + + @markers.aws.validated + def test_bucket_acceleration_configuration_exc( + self, s3_bucket, s3_create_bucket, snapshot, aws_client + ): + with pytest.raises(ClientError) as e: + aws_client.s3.put_bucket_accelerate_configuration( + Bucket=s3_bucket, + AccelerateConfiguration={"Status": "enabled"}, + ) + snapshot.match("put-bucket-accelerate-config-lowercase", e.value.response) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_bucket_accelerate_configuration( + Bucket=s3_bucket, + AccelerateConfiguration={"Status": "random"}, + ) + snapshot.match("put-bucket-accelerate-config-random", e.value.response) + + bucket_with_name = s3_create_bucket(Bucket=f"test.bucket.{long_uid()}") + with pytest.raises(ClientError) as e: + aws_client.s3.put_bucket_accelerate_configuration( + Bucket=bucket_with_name, + AccelerateConfiguration={"Status": "random"}, + ) + snapshot.match("put-bucket-accelerate-config-dot-bucket", e.value.response) diff --git a/tests/aws/s3/test_s3_api.snapshot.json b/tests/aws/s3/test_s3_api.snapshot.json index 048d5e130f6df..98c26c85015ab 100644 --- a/tests/aws/s3/test_s3_api.snapshot.json +++ b/tests/aws/s3/test_s3_api.snapshot.json @@ -2497,5 +2497,170 @@ "tests/aws/s3/test_s3_api.py::TestS3PublicAccessBlock::test_public_access_block_exc": { "recorded-date": "10-08-2023, 03:30:54", "recorded-content": {} + }, + "tests/aws/s3/test_s3_api.py::TestS3BucketPolicy::test_bucket_policy_crud": { + "recorded-date": "10-08-2023, 17:27:26", + "recorded-content": { + "get-bucket-default-policy": { + "Error": { + "BucketName": "