From 04160b82361157f71364eb0f4aea4a19d76e0323 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Sun, 13 Aug 2023 02:39:25 +0200 Subject: [PATCH 1/3] implement ACL --- localstack/services/s3/constants.py | 29 +- localstack/services/s3/provider.py | 12 +- localstack/services/s3/utils.py | 60 +++- localstack/services/s3/v3/models.py | 32 ++- localstack/services/s3/v3/provider.py | 288 ++++++++++++++++---- localstack/services/s3/validation.py | 64 ++++- tests/aws/services/s3/test_s3.py | 32 ++- tests/aws/services/s3/test_s3.snapshot.json | 50 +++- tests/unit/test_s3.py | 69 +++-- 9 files changed, 487 insertions(+), 149 deletions(-) diff --git a/localstack/services/s3/constants.py b/localstack/services/s3/constants.py index 4cbe90108c607..6887b43e1e972 100644 --- a/localstack/services/s3/constants.py +++ b/localstack/services/s3/constants.py @@ -1,6 +1,5 @@ from localstack.aws.api.s3 import ( - BucketCannedACL, - ObjectCannedACL, + Grantee, Permission, PublicAccessBlockConfiguration, ServerSideEncryption, @@ -8,6 +7,7 @@ ServerSideEncryptionRule, StorageClass, ) +from localstack.aws.api.s3 import Type as GranteeType S3_VIRTUAL_HOST_FORWARDED_HEADER = "x-s3-vhost-forwarded-for" @@ -16,23 +16,14 @@ This is minimum size allowed by S3 when uploading more than one part for a Multipart Upload, except for the last part """ -VALID_CANNED_ACLS_BUCKET = { - # https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl - # bucket-owner-read + bucket-owner-full-control are allowed, but ignored for buckets - ObjectCannedACL.private, - ObjectCannedACL.authenticated_read, - ObjectCannedACL.aws_exec_read, - ObjectCannedACL.bucket_owner_full_control, - ObjectCannedACL.bucket_owner_read, - ObjectCannedACL.public_read, - ObjectCannedACL.public_read_write, - BucketCannedACL.log_delivery_write, -} +AUTHENTICATED_USERS_ACL_GROUP = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" +ALL_USERS_ACL_GROUP = "http://acs.amazonaws.com/groups/global/AllUsers" +LOG_DELIVERY_ACL_GROUP = "http://acs.amazonaws.com/groups/s3/LogDelivery" VALID_ACL_PREDEFINED_GROUPS = { - "http://acs.amazonaws.com/groups/global/AuthenticatedUsers", - "http://acs.amazonaws.com/groups/global/AllUsers", - "http://acs.amazonaws.com/groups/s3/LogDelivery", + AUTHENTICATED_USERS_ACL_GROUP, + ALL_USERS_ACL_GROUP, + LOG_DELIVERY_ACL_GROUP, } VALID_GRANTEE_PERMISSIONS = { @@ -117,3 +108,7 @@ RestrictPublicBuckets=True, IgnorePublicAcls=True, ) + +AUTHENTICATED_USERS_ACL_GRANTEE = Grantee(URI=AUTHENTICATED_USERS_ACL_GROUP, Type=GranteeType.Group) +ALL_USERS_ACL_GRANTEE = Grantee(URI=ALL_USERS_ACL_GROUP, Type=GranteeType.Group) +LOG_DELIVERY_ACL_GRANTEE = Grantee(URI=LOG_DELIVERY_ACL_GROUP, Type=GranteeType.Group) diff --git a/localstack/services/s3/provider.py b/localstack/services/s3/provider.py index ceda0ebd3173f..29b1b97f98fe5 100644 --- a/localstack/services/s3/provider.py +++ b/localstack/services/s3/provider.py @@ -170,18 +170,19 @@ get_key_from_moto_bucket, get_lifecycle_rule_from_object, get_object_checksum_for_algorithm, + get_permission_from_header, is_key_expired, serialize_expiration_header, validate_kms_key_id, verify_checksum, ) from localstack.services.s3.validation import ( + parse_grants_in_headers, validate_acl_acp, validate_bucket_analytics_configuration, validate_bucket_intelligent_tiering_configuration, validate_bucket_name, validate_canned_acl, - validate_grantee_in_headers, validate_inventory_configuration, validate_lifecycle_configuration, validate_website_configuration, @@ -1183,8 +1184,10 @@ def put_bucket_acl( validate_canned_acl(canned_acl) elif present_headers: - for key, grantees_values in present_headers: - validate_grantee_in_headers(key, grantees_values) + for key in grant_keys: + if grantees_values := request.get(key, ""): # noqa + permission = get_permission_from_header(key) + parse_grants_in_headers(permission, grantees_values) elif acp := request.get("AccessControlPolicy"): validate_acl_acp(acp) @@ -1227,7 +1230,8 @@ def put_object_acl( ] for key in grant_keys: if grantees_values := request.get(key, ""): # noqa - validate_grantee_in_headers(key, grantees_values) + permission = get_permission_from_header(key) + parse_grants_in_headers(permission, grantees_values) if acp := request.get("AccessControlPolicy"): validate_acl_acp(acp) diff --git a/localstack/services/s3/utils.py b/localstack/services/s3/utils.py index 2ef49a36f3982..46ad8e2cd6d39 100644 --- a/localstack/services/s3/utils.py +++ b/localstack/services/s3/utils.py @@ -17,12 +17,16 @@ from localstack import config from localstack.aws.api import CommonServiceException, RequestContext from localstack.aws.api.s3 import ( + AccessControlPolicy, + BucketCannedACL, BucketName, ChecksumAlgorithm, CopyObjectRequest, CopySource, ETag, GetObjectRequest, + Grant, + Grantee, HeadObjectRequest, InvalidArgument, InvalidRange, @@ -34,23 +38,28 @@ MethodNotAllowed, NoSuchBucket, NoSuchKey, + ObjectCannedACL, ObjectKey, ObjectSize, ObjectVersionId, Owner, + Permission, PreconditionFailed, SSEKMSKeyId, TaggingHeader, TagSet, ) +from localstack.aws.api.s3 import Type as GranteeType from localstack.aws.connect import connect_to from localstack.services.s3.constants import ( + ALL_USERS_ACL_GRANTEE, + AUTHENTICATED_USERS_ACL_GRANTEE, + LOG_DELIVERY_ACL_GRANTEE, S3_CHUNK_SIZE, S3_VIRTUAL_HOST_FORWARDED_HEADER, SIGNATURE_V2_PARAMS, SIGNATURE_V4_PARAMS, SYSTEM_METADATA_SETTABLE_HEADERS, - VALID_CANNED_ACLS_BUCKET, ) from localstack.services.s3.exceptions import InvalidRequest from localstack.utils.aws import arns @@ -325,13 +334,13 @@ def is_bucket_name_valid(bucket_name: str) -> bool: return True if re.match(BUCKET_NAME_REGEX, bucket_name) else False -def is_canned_acl_bucket_valid(canned_acl: str) -> bool: - return canned_acl in VALID_CANNED_ACLS_BUCKET +def get_permission_header_name(permission: Permission) -> str: + return f"x-amz-grant-{permission.replace('_', '-').lower()}" -def get_header_name(capitalized_field: str) -> str: - headers_parts = re.split(r"([A-Z][a-z]+)", capitalized_field) - return f"x-amz-{'-'.join([part.lower() for part in headers_parts if part])}" +def get_permission_from_header(capitalized_field: str) -> Permission: + headers_parts = [part.upper() for part in re.split(r"([A-Z][a-z]+)", capitalized_field) if part] + return "_".join(headers_parts[1:]) def is_valid_canonical_id(canonical_id: str) -> bool: @@ -860,3 +869,42 @@ def validate_failed_precondition( code="NotModified", status_code=304, ) + + +def get_canned_acl( + canned_acl: BucketCannedACL | ObjectCannedACL, owner: Owner +) -> AccessControlPolicy: + """ + Return the proper Owner and Grants from a CannedACL + See https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl + :param canned_acl: an S3 CannedACL + :param owner: the current owner of the bucket or object + :return: an AccessControlPolicy containing the Grants and Owner + """ + owner_grantee = Grantee(**owner, Type=GranteeType.CanonicalUser) + grants = [Grant(Grantee=owner_grantee, Permission=Permission.FULL_CONTROL)] + + match canned_acl: + case ObjectCannedACL.private: + pass # no other permissions + case ObjectCannedACL.public_read: + grants.append(Grant(Grantee=ALL_USERS_ACL_GRANTEE, Permission=Permission.READ)) + + case ObjectCannedACL.public_read_write: + grants.append(Grant(Grantee=ALL_USERS_ACL_GRANTEE, Permission=Permission.READ)) + grants.append(Grant(Grantee=ALL_USERS_ACL_GRANTEE, Permission=Permission.WRITE)) + case ObjectCannedACL.authenticated_read: + grants.append( + Grant(Grantee=AUTHENTICATED_USERS_ACL_GRANTEE, Permission=Permission.READ) + ) + case ObjectCannedACL.bucket_owner_read: + pass # TODO: bucket owner ACL + case ObjectCannedACL.bucket_owner_full_control: + pass # TODO: bucket owner ACL + case ObjectCannedACL.aws_exec_read: + pass # TODO: bucket owner, EC2 Read + case BucketCannedACL.log_delivery_write: + grants.append(Grant(Grantee=LOG_DELIVERY_ACL_GRANTEE, Permission=Permission.READ_ACP)) + grants.append(Grant(Grantee=LOG_DELIVERY_ACL_GRANTEE, Permission=Permission.WRITE)) + + return AccessControlPolicy(Owner=owner, Grants=grants) diff --git a/localstack/services/s3/v3/models.py b/localstack/services/s3/v3/models.py index 43c35fff5acfb..37732ced5466a 100644 --- a/localstack/services/s3/v3/models.py +++ b/localstack/services/s3/v3/models.py @@ -9,6 +9,7 @@ from localstack import config from localstack.aws.api import CommonServiceException from localstack.aws.api.s3 import ( + AccessControlPolicy, AccountId, AnalyticsConfiguration, AnalyticsId, @@ -67,7 +68,6 @@ S3_UPLOAD_PART_MIN_SIZE, ) from localstack.services.s3.utils import ( - get_owner_for_account_id, iso_8601_datetime_without_milliseconds_s3, rfc_1123_datetime, ) @@ -101,7 +101,7 @@ class S3Bucket: lifecycle_rules: Optional[LifecycleRules] policy: Optional[Policy] website_configuration: Optional[WebsiteConfiguration] - acl: str # TODO: change this + acl: AccessControlPolicy cors_rules: Optional[CORSConfiguration] logging: LoggingEnabled notification_configuration: NotificationConfiguration @@ -124,7 +124,8 @@ def __init__( name: BucketName, account_id: AccountId, bucket_region: BucketRegion, - acl=None, # TODO: validate ACL first, create utils for validating and consolidating + owner: Owner, + acl: AccessControlPolicy = None, object_ownership: ObjectOwnership = None, object_lock_enabled_for_bucket: bool = None, ): @@ -153,9 +154,9 @@ def __init__( self.inventory_configurations = {} self.object_lock_default_retention = {} self.replication = None - + self.acl = acl # see https://docs.aws.amazon.com/AmazonS3/latest/API/API_Owner.html - self.owner = get_owner_for_account_id(account_id) + self.owner = owner self.bucket_arn = arns.s3_bucket_arn(self.name) def get_object( @@ -225,6 +226,16 @@ def get_object( raise NoSuchKey("The specified key does not exist.", Key=key) elif raise_for_delete_marker and isinstance(s3_object, S3DeleteMarker): + if http_method not in ("HEAD", "GET"): + raise MethodNotAllowed( + "The specified method is not allowed against this resource.", + Method=http_method, + ResourceType="DeleteMarker", + DeleteMarker=True, + Allow="DELETE", + VersionId=s3_object.version_id, + ) + raise NoSuchKey( "The specified key does not exist.", Key=key, @@ -240,6 +251,7 @@ class S3Object: key: ObjectKey version_id: Optional[ObjectVersionId] bucket: BucketName + owner: Optional[Owner] size: Optional[Size] etag: Optional[ETag] user_metadata: Metadata @@ -257,7 +269,7 @@ class S3Object: lock_legal_status: Optional[ObjectLockLegalHoldStatus] lock_until: Optional[datetime] website_redirect_location: Optional[WebsiteRedirectLocation] - acl: Optional[str] # TODO: we need to change something here, how it's done? + acl: Optional[AccessControlPolicy] is_current: bool parts: Optional[dict[int, tuple[int, int]]] restore: Optional[Restore] @@ -282,7 +294,8 @@ def __init__( lock_legal_status: Optional[ObjectLockLegalHoldStatus] = None, lock_until: Optional[datetime] = None, website_redirect_location: Optional[WebsiteRedirectLocation] = None, - acl: Optional[str] = None, # TODO + acl: Optional[AccessControlPolicy] = None, # TODO + owner: Optional[Owner] = None, ): self.key = key self.user_metadata = ( @@ -309,6 +322,7 @@ def __init__( self.last_modified = datetime.now(tz=_gmt_zone_info) self.parts = {} self.restore = None + self.owner = owner def get_system_metadata_fields(self) -> dict: headers = { @@ -433,11 +447,12 @@ def __init__( lock_legal_status: Optional[ObjectLockLegalHoldStatus] = None, lock_until: Optional[datetime] = None, website_redirect_location: Optional[WebsiteRedirectLocation] = None, - acl: Optional[str] = None, # TODO + acl: Optional[AccessControlPolicy] = None, # TODO user_metadata: Optional[Metadata] = None, system_metadata: Optional[Metadata] = None, initiator: Optional[Owner] = None, tagging: Optional[dict[str, str]] = None, + owner: Optional[Owner] = None, ): self.id = token_urlsafe(96) # MultipartUploadId is 128 characters long self.initiated = datetime.now(tz=_gmt_zone_info) @@ -461,6 +476,7 @@ def __init__( lock_until=lock_until, website_redirect_location=website_redirect_location, acl=acl, + owner=owner, ) def complete_multipart(self, parts: CompletedPartList): diff --git a/localstack/services/s3/v3/provider.py b/localstack/services/s3/v3/provider.py index d17e2b3a816d2..e22111e045774 100644 --- a/localstack/services/s3/v3/provider.py +++ b/localstack/services/s3/v3/provider.py @@ -6,6 +6,7 @@ from collections import defaultdict from operator import itemgetter from secrets import token_urlsafe +from typing import Union from localstack import config from localstack.aws.api import CommonServiceException, RequestContext, handler @@ -60,6 +61,7 @@ Expiration, FetchOwner, GetBucketAccelerateConfigurationOutput, + GetBucketAclOutput, GetBucketAnalyticsConfigurationOutput, GetBucketCorsOutput, GetBucketEncryptionOutput, @@ -70,11 +72,13 @@ GetBucketLoggingOutput, GetBucketOwnershipControlsOutput, GetBucketPolicyOutput, + GetBucketPolicyStatusOutput, GetBucketReplicationOutput, GetBucketRequestPaymentOutput, GetBucketTaggingOutput, GetBucketVersioningOutput, GetBucketWebsiteOutput, + GetObjectAclOutput, GetObjectAttributesOutput, GetObjectAttributesParts, GetObjectAttributesRequest, @@ -84,12 +88,8 @@ GetObjectRequest, GetObjectRetentionOutput, GetObjectTaggingOutput, + GetObjectTorrentOutput, GetPublicAccessBlockOutput, - GrantFullControl, - GrantRead, - GrantReadACP, - GrantWrite, - GrantWriteACP, HeadBucketOutput, HeadObjectOutput, HeadObjectRequest, @@ -119,6 +119,7 @@ MaxKeys, MaxParts, MaxUploads, + MissingSecurityHeader, MultipartUpload, MultipartUploadId, NoSuchBucket, @@ -146,6 +147,7 @@ ObjectVersionId, ObjectVersionStorageClass, OptionalObjectAttributesList, + Owner, OwnershipControls, OwnershipControlsNotFoundError, Part, @@ -155,6 +157,9 @@ PreconditionFailed, Prefix, PublicAccessBlockConfiguration, + PutBucketAclRequest, + PutObjectAclOutput, + PutObjectAclRequest, PutObjectLegalHoldOutput, PutObjectLockConfigurationOutput, PutObjectOutput, @@ -205,6 +210,7 @@ MalformedXML, NoSuchConfiguration, NoSuchObjectLockConfiguration, + UnexpectedContent, ) from localstack.services.s3.notifications import NotificationDispatcher, S3EventNotificationContext from localstack.services.s3.utils import ( @@ -212,12 +218,14 @@ add_expiration_days_to_datetime, create_s3_kms_managed_key_for_region, extract_bucket_key_version_id_from_copy_source, + get_canned_acl, get_class_attrs_from_spec_class, get_failed_precondition_copy_source, get_full_default_bucket_location, get_kms_key_arn, get_lifecycle_rule_from_object, get_owner_for_account_id, + get_permission_from_header, get_retention_from_now, get_system_metadata_from_request, get_unique_key_id, @@ -246,8 +254,11 @@ from localstack.services.s3.v3.storage.core import LimitedIterableStream from localstack.services.s3.v3.storage.ephemeral import EphemeralS3ObjectStore from localstack.services.s3.validation import ( + parse_grants_in_headers, + validate_acl_acp, validate_bucket_analytics_configuration, validate_bucket_intelligent_tiering_configuration, + validate_canned_acl, validate_cors_configuration, validate_inventory_configuration, validate_lifecycle_configuration, @@ -417,15 +428,19 @@ def create_bucket( f"Invalid x-amz-object-ownership header: {object_ownership}", ArgumentName="x-amz-object-ownership", ) - + # see https://docs.aws.amazon.com/AmazonS3/latest/API/API_Owner.html + owner = get_owner_for_account_id(context.account_id) + acl = get_access_control_policy_for_new_resource_request(request, owner=owner) s3_bucket = S3Bucket( name=bucket_name, account_id=context.account_id, bucket_region=bucket_region, - acl=None, # TODO: validate ACL first, create utils for validating and consolidating + owner=owner, + acl=acl, object_ownership=request.get("ObjectOwnership"), object_lock_enabled_for_bucket=request.get("ObjectLockEnabledForBucket"), ) + store.buckets[bucket_name] = s3_bucket store.global_bucket_map[bucket_name] = s3_bucket.bucket_account_id self._cors_handler.invalidate_cache() @@ -520,13 +535,7 @@ def put_object( request: PutObjectRequest, ) -> PutObjectOutput: # TODO: validate order of validation - # TODO: still need to handle following parameters: - # acl: ObjectCannedACL = None, - # grant_full_control: GrantFullControl = None, - # grant_read: GrantRead = None, - # grant_read_acp: GrantReadACP = None, - # grant_write_acp: GrantWriteACP = None, - # - + # TODO: still need to handle following parameters # request_payer: RequestPayer = None, bucket_name = request["Bucket"] store, s3_bucket = self._get_cross_account_bucket(context, bucket_name) @@ -547,9 +556,6 @@ def put_object( if not system_metadata.get("ContentType"): system_metadata["ContentType"] = "binary/octet-stream" - # TODO: get all default from bucket once it is implemented - # validate encryption values - body = request.get("Body") # check if chunked request headers = context.request.headers @@ -573,6 +579,8 @@ def put_object( lock_parameters = get_object_lock_parameters_from_bucket_and_request(request, s3_bucket) + acl = get_access_control_policy_for_new_resource_request(request, owner=s3_bucket.owner) + s3_object = S3Object( key=key, version_id=version_id, @@ -589,8 +597,8 @@ def put_object( lock_legal_status=lock_parameters.lock_legal_status, lock_until=lock_parameters.lock_until, website_redirect_location=request.get("WebsiteRedirectLocation"), - expiration=None, # TODO, from lifecycle, or should it be updated with config? - acl=None, + acl=acl, + owner=s3_bucket.owner, # TODO: for now we only have one owner, but it can depends on Bucket settings ) s3_stored_object = self._storage_backend.open(bucket_name, s3_object) @@ -1126,6 +1134,10 @@ def copy_object( request, dest_s3_bucket ) + acl = get_access_control_policy_for_new_resource_request( + request, owner=dest_s3_bucket.owner + ) + s3_object = S3Object( key=dest_key, size=src_s3_object.size, @@ -1145,7 +1157,8 @@ def copy_object( lock_until=lock_parameters.lock_until, website_redirect_location=website_redirect_location, expiration=None, # TODO, from lifecycle - acl=None, + acl=acl, + owner=dest_s3_bucket.owner, ) s3_stored_object = self._storage_backend.copy( @@ -1635,11 +1648,6 @@ def create_multipart_upload( request: CreateMultipartUploadRequest, ) -> CreateMultipartUploadOutput: # TODO: handle missing parameters: - # acl: ObjectCannedACL = None, - # grant_full_control: GrantFullControl = None, - # grant_read: GrantRead = None, - # grant_read_acp: GrantReadACP = None, - # grant_write_acp: GrantWriteACP = None, # request_payer: RequestPayer = None, store = self.get_store(context.account_id, context.region) bucket_name = request["Bucket"] @@ -1665,10 +1673,6 @@ def create_multipart_upload( if not system_metadata.get("ContentType"): system_metadata["ContentType"] = "binary/octet-stream" - # TODO: get all default from bucket, maybe extract logic - - # TODO: consolidate ACL into one, and validate it - # TODO: validate the algorithm? checksum_algorithm = request.get("ChecksumAlgorithm") @@ -1679,6 +1683,8 @@ def create_multipart_upload( ) lock_parameters = get_object_lock_parameters_from_bucket_and_request(request, s3_bucket) + acl = get_access_control_policy_for_new_resource_request(request, owner=s3_bucket.owner) + # validate encryption values s3_multipart = S3Multipart( key=key, @@ -1695,9 +1701,10 @@ def create_multipart_upload( lock_until=lock_parameters.lock_until, website_redirect_location=request.get("WebsiteRedirectLocation"), expiration=None, # TODO, from lifecycle, or should it be updated with config? - acl=None, + acl=acl, initiator=get_owner_for_account_id(context.account_id), tagging=tagging, + owner=s3_bucket.owner, ) s3_bucket.multiparts[s3_multipart.id] = s3_multipart @@ -1740,16 +1747,17 @@ def upload_part( "The upload ID may be invalid, or the upload may have been aborted or completed.", UploadId=upload_id, ) + elif (part_number := request.get("PartNumber", 0)) < 1 or part_number > 10000: + raise InvalidArgument( + "Part number must be an integer between 1 and 10000, inclusive", + ArgumentName="partNumber", + ArgumentValue=part_number, + ) # TODO: validate key? if s3_multipart.object.key != request.get("Key"): pass - part_number = request.get("PartNumber") - # TODO: validate PartNumber - # if part_number > 10000: - # raise InvalidMaxPartNumberArgument(part_number) - body = request.get("Body") headers = context.request.headers is_aws_chunked = headers.get("x-amz-content-sha256", "").startswith("STREAMING-") @@ -1833,15 +1841,17 @@ def upload_part_copy( UploadId=upload_id, ) + elif (part_number := request.get("PartNumber", 0)) < 1 or part_number > 10000: + raise InvalidArgument( + "Part number must be an integer between 1 and 10000, inclusive", + ArgumentName="partNumber", + ArgumentValue=part_number, + ) + # TODO: validate key? if s3_multipart.object.key != dest_key: pass - part_number = request.get("PartNumber") - # TODO: validate PartNumber - # if part_number > 10000: - # raise InvalidMaxPartNumberArgument(part_number) - source_range = request.get("CopySourceRange") # TODO implement copy source IF (done in ASF provider) range_data = parse_range_header(source_range, src_s3_object.size) @@ -3258,25 +3268,92 @@ def delete_bucket_replication( s3_bucket.replication = None - # ###### THIS ARE UNIMPLEMENTED METHODS TO ALLOW TESTING, DO NOT COUNT THEM AS DONE ###### # - + @handler("PutBucketAcl", expand=False) def put_bucket_acl( + self, + context: RequestContext, + request: PutBucketAclRequest, + ) -> None: + bucket = request["Bucket"] + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + acp = get_access_control_policy_from_acl_request( + request=request, owner=s3_bucket.owner, request_body=context.request.data + ) + s3_bucket.acl = acp + + def get_bucket_acl( + self, context: RequestContext, bucket: BucketName, expected_bucket_owner: AccountId = None + ) -> GetBucketAclOutput: + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + + return GetBucketAclOutput(Owner=s3_bucket.acl["Owner"], Grants=s3_bucket.acl["Grants"]) + + @handler("PutObjectAcl", expand=False) + def put_object_acl( + self, + context: RequestContext, + request: PutObjectAclRequest, + ) -> PutObjectAclOutput: + bucket = request["Bucket"] + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + + s3_object = s3_bucket.get_object( + key=request["Key"], + version_id=request.get("VersionId"), + http_method="PUT", + ) + acp = get_access_control_policy_from_acl_request( + request=request, owner=s3_object.owner, request_body=context.request.data + ) + previous_acl = s3_object.acl + s3_object.acl = acp + + if previous_acl != acp: + self._notify(context, s3_bucket=s3_bucket, s3_object=s3_object) + + # TODO: RequestCharged + return PutObjectAclOutput() + + def get_object_acl( self, context: RequestContext, bucket: BucketName, - acl: BucketCannedACL = None, - access_control_policy: AccessControlPolicy = None, - content_md5: ContentMD5 = None, - checksum_algorithm: ChecksumAlgorithm = None, - grant_full_control: GrantFullControl = None, - grant_read: GrantRead = None, - grant_read_acp: GrantReadACP = None, - grant_write: GrantWrite = None, - grant_write_acp: GrantWriteACP = None, + key: ObjectKey, + version_id: ObjectVersionId = None, + request_payer: RequestPayer = None, expected_bucket_owner: AccountId = None, - ) -> None: - # TODO: implement, this is just for CORS tests to be able to run - pass + ) -> GetObjectAclOutput: + store = self.get_store(context.account_id, context.region) + if not (s3_bucket := store.buckets.get(bucket)): + raise NoSuchBucket("The specified bucket does not exist", BucketName=bucket) + + s3_object = s3_bucket.get_object( + key=key, + version_id=version_id, + ) + # TODO: RequestCharged + return GetObjectAclOutput(Owner=s3_object.acl["Owner"], Grants=s3_object.acl["Grants"]) + + def get_bucket_policy_status( + self, context: RequestContext, bucket: BucketName, expected_bucket_owner: AccountId = None + ) -> GetBucketPolicyStatusOutput: + raise NotImplementedError + + def get_object_torrent( + self, + context: RequestContext, + bucket: BucketName, + key: ObjectKey, + request_payer: RequestPayer = None, + expected_bucket_owner: AccountId = None, + ) -> GetObjectTorrentOutput: + raise NotImplementedError def generate_version_id(bucket_versioning_status: str) -> str | None: @@ -3383,3 +3460,106 @@ def get_part_range(s3_object: S3Object, part_number: PartNumber) -> ObjectRange: content_length=part_length, content_range=f"bytes {begin}-{end}/{s3_object.size}", ) + + +def get_acl_headers_from_request( + request: Union[ + PutObjectRequest, + CreateMultipartUploadRequest, + CopyObjectRequest, + CreateBucketRequest, + PutBucketAclRequest, + PutObjectAclRequest, + ] +) -> list[tuple[str, str]]: + permission_keys = [ + "GrantFullControl", + "GrantRead", + "GrantReadACP", + "GrantWrite", + "GrantWriteACP", + ] + acl_headers = [ + (permission, grant_header) + for permission in permission_keys + if (grant_header := request.get(permission)) + ] + return acl_headers + + +def get_access_control_policy_from_acl_request( + request: Union[PutBucketAclRequest, PutObjectAclRequest], + owner: Owner, + request_body: bytes, +) -> AccessControlPolicy: + canned_acl = request.get("ACL") + acl_headers = get_acl_headers_from_request(request) + + # FIXME: this is very dirty, but the parser does not differentiate between an empty body and an empty XML node + # errors are different depending on that data, so we need to access the context. Modifying the parser for this + # use case seems dangerous + is_acp_in_body = request_body + + if not (canned_acl or acl_headers or is_acp_in_body): + raise MissingSecurityHeader( + "Your request was missing a required header", MissingHeaderName="x-amz-acl" + ) + + elif canned_acl and acl_headers: + raise InvalidRequest("Specifying both Canned ACLs and Header Grants is not allowed") + + elif (canned_acl or acl_headers) and is_acp_in_body: + raise UnexpectedContent("This request does not support content") + + if canned_acl: + validate_canned_acl(canned_acl) + acp = get_canned_acl(canned_acl, owner=owner) + + elif acl_headers: + grants = [] + for permission, grantees_values in acl_headers: + permission = get_permission_from_header(permission) + partial_grants = parse_grants_in_headers(permission, grantees_values) + grants.extend(partial_grants) + + acp = AccessControlPolicy(Owner=owner, Grants=grants) + else: + acp = request.get("AccessControlPolicy") + validate_acl_acp(acp) + if ( + owner.get("DisplayName") + and acp["Grants"] + and "DisplayName" not in acp["Grants"][0]["Grantee"] + ): + acp["Grants"][0]["Grantee"]["DisplayName"] = owner["DisplayName"] + + return acp + + +def get_access_control_policy_for_new_resource_request( + request: Union[ + PutObjectRequest, CreateMultipartUploadRequest, CopyObjectRequest, CreateBucketRequest + ], + owner: Owner, +) -> AccessControlPolicy: + # TODO: this is basic ACL, not taking into account Bucket settings. Revisit once we really implement ACLs. + canned_acl = request.get("ACL") + acl_headers = get_acl_headers_from_request(request) + + if not (canned_acl or acl_headers): + return get_canned_acl(BucketCannedACL.private, owner=owner) + + elif canned_acl and acl_headers: + raise InvalidRequest("Specifying both Canned ACLs and Header Grants is not allowed") + + if canned_acl: + validate_canned_acl(canned_acl) + return get_canned_acl(canned_acl, owner=owner) + + grants = [] + for permission, grantees_values in acl_headers: + permission = get_permission_from_header(permission) + partial_grants = parse_grants_in_headers(permission, grantees_values) + grants.extend(partial_grants) + + return AccessControlPolicy(Owner=owner, Grants=grants) diff --git a/localstack/services/s3/validation.py b/localstack/services/s3/validation.py index 11a27e8dc6b14..2278033f33e30 100644 --- a/localstack/services/s3/validation.py +++ b/localstack/services/s3/validation.py @@ -8,15 +8,21 @@ AccessControlPolicy, AnalyticsConfiguration, AnalyticsId, + BucketCannedACL, BucketLifecycleConfiguration, BucketName, CORSConfiguration, + Grant, + Grantee, + Grants, IntelligentTieringConfiguration, IntelligentTieringId, InvalidArgument, InvalidBucketName, InventoryConfiguration, InventoryId, + ObjectCannedACL, + Permission, ) from localstack.aws.api.s3 import Type as GranteeType from localstack.aws.api.s3 import WebsiteConfiguration @@ -24,14 +30,20 @@ from localstack.services.s3.exceptions import InvalidRequest, MalformedACLError, MalformedXML from localstack.services.s3.utils import ( _create_invalid_argument_exc, - get_header_name, + get_class_attrs_from_spec_class, + get_permission_header_name, is_bucket_name_valid, - is_canned_acl_bucket_valid, is_valid_canonical_id, validate_dict_fields, ) from localstack.utils.aws import arns +# https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl +# bucket-owner-read + bucket-owner-full-control are allowed, but ignored for buckets +VALID_CANNED_ACLS = get_class_attrs_from_spec_class( + BucketCannedACL +) | get_class_attrs_from_spec_class(ObjectCannedACL) + def validate_bucket_analytics_configuration( id: AnalyticsId, analytics_configuration: AnalyticsConfiguration @@ -64,30 +76,52 @@ def validate_canned_acl(canned_acl: str) -> None: """ Validate the canned ACL value, or raise an Exception """ - if canned_acl and not is_canned_acl_bucket_valid(canned_acl): + if canned_acl and canned_acl not in VALID_CANNED_ACLS: ex = _create_invalid_argument_exc(None, "x-amz-acl", canned_acl) raise ex -def validate_grantee_in_headers(grant: str, grantees: str) -> None: +def parse_grants_in_headers(permission: Permission, grantees: str) -> Grants: splitted_grantees = [grantee.strip() for grantee in grantees.split(",")] - for grantee in splitted_grantees: - grantee_type, grantee_id = grantee.split("=") + grants = [] + for seralized_grantee in splitted_grantees: + grantee_type, grantee_id = seralized_grantee.split("=") grantee_id = grantee_id.strip('"') if grantee_type not in ("uri", "id", "emailAddress"): ex = _create_invalid_argument_exc( - "Argument format not recognized", get_header_name(grant), grantee + "Argument format not recognized", + get_permission_header_name(permission), + seralized_grantee, ) raise ex - elif grantee_type == "uri" and grantee_id not in s3_constants.VALID_ACL_PREDEFINED_GROUPS: - ex = _create_invalid_argument_exc("Invalid group uri", "uri", grantee_id) - raise ex - elif grantee_type == "id" and not is_valid_canonical_id(grantee_id): - ex = _create_invalid_argument_exc("Invalid id", "id", grantee_id) - raise ex - elif grantee_type == "emailAddress": + elif grantee_type == "uri": + if grantee_id not in s3_constants.VALID_ACL_PREDEFINED_GROUPS: + ex = _create_invalid_argument_exc("Invalid group uri", "uri", grantee_id) + raise ex + grantee = Grantee( + Type=GranteeType.Group, + URI=grantee_id, + ) + + elif grantee_type == "id": + if not is_valid_canonical_id(grantee_id): + ex = _create_invalid_argument_exc("Invalid id", "id", grantee_id) + raise ex + grantee = Grantee( + Type=GranteeType.CanonicalUser, + ID=grantee_id, + DisplayName="webfile", # TODO: only in certain regions + ) + + else: # TODO: check validation here - continue + grantee = Grantee( + Type=GranteeType.AmazonCustomerByEmail, + EmailAddress=grantee_id, + ) + grants.append(Grant(Permission=permission, Grantee=grantee)) + + return grants def validate_acl_acp(acp: AccessControlPolicy) -> None: diff --git a/tests/aws/services/s3/test_s3.py b/tests/aws/services/s3/test_s3.py index dff3b0f5272d6..4b2a8174a751f 100644 --- a/tests/aws/services/s3/test_s3.py +++ b/tests/aws/services/s3/test_s3.py @@ -2832,6 +2832,8 @@ def test_delete_non_existing_keys_in_non_existing_bucket(self, snapshot, aws_cli "$..ServerSideEncryption", "$..Deleted..DeleteMarker", "$..Deleted..DeleteMarkerVersionId", + "$.get-acl-delete-marker-version-id.Error", # Moto is not handling that case well with versioning + "$.get-acl-delete-marker-version-id.ResponseMetadata", ], ) @markers.snapshot.skip_snapshot_verify(condition=is_old_provider, paths=["$..VersionId"]) @@ -2849,18 +2851,32 @@ def test_put_object_acl_on_delete_marker( put_obj_2 = aws_client.s3.put_object(Bucket=s3_bucket, Key=object_key, Body="something-v2") snapshot.match("put-obj-2", put_obj_2) - # delete objects - response = aws_client.s3.delete_objects( - Bucket=s3_bucket, - Delete={ - "Objects": [{"Key": object_key}], - }, - ) + response = aws_client.s3.delete_object(Bucket=s3_bucket, Key=object_key) snapshot.match("delete-object", response) + delete_marker_version_id = response["VersionId"] with pytest.raises(ClientError) as e: aws_client.s3.put_object_acl(Bucket=s3_bucket, Key=object_key, ACL="public-read") - snapshot.match("key-delete-marker", e.value.response) + snapshot.match("put-acl-delete-marker", e.value.response) + + with pytest.raises(ClientError) as e: + aws_client.s3.get_object_acl(Bucket=s3_bucket, Key=object_key) + snapshot.match("get-acl-delete-marker", e.value.response) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl( + Bucket=s3_bucket, + Key=object_key, + VersionId=delete_marker_version_id, + ACL="public-read", + ) + snapshot.match("put-acl-delete-marker-version-id", e.value.response) + + with pytest.raises(ClientError) as e: + aws_client.s3.get_object_acl( + Bucket=s3_bucket, Key=object_key, VersionId=delete_marker_version_id + ) + snapshot.match("get-acl-delete-marker-version-id", e.value.response) @markers.aws.validated def test_s3_request_payer(self, s3_bucket, snapshot, aws_client): diff --git a/tests/aws/services/s3/test_s3.snapshot.json b/tests/aws/services/s3/test_s3.snapshot.json index 0825526fdc121..10db14faf0b0e 100644 --- a/tests/aws/services/s3/test_s3.snapshot.json +++ b/tests/aws/services/s3/test_s3.snapshot.json @@ -5373,7 +5373,7 @@ } }, "tests/aws/services/s3/test_s3.py::TestS3::test_put_object_acl_on_delete_marker": { - "recorded-date": "04-08-2023, 23:53:09", + "recorded-date": "13-08-2023, 02:27:00", "recorded-content": { "put-obj-1": { "ETag": "\"437b930db84b8079c2dd804a71936b5f\"", @@ -5394,19 +5394,14 @@ } }, "delete-object": { - "Deleted": [ - { - "DeleteMarker": true, - "DeleteMarkerVersionId": "PY9ezDfMH8Xgs0AK1ri_1goNuAwSqcbA", - "Key": "test-key-versioned" - } - ], + "DeleteMarker": true, + "VersionId": "", "ResponseMetadata": { "HTTPHeaders": {}, - "HTTPStatusCode": 200 + "HTTPStatusCode": 204 } }, - "key-delete-marker": { + "put-acl-delete-marker": { "Error": { "Code": "MethodNotAllowed", "Message": "The specified method is not allowed against this resource.", @@ -5417,6 +5412,41 @@ "HTTPHeaders": {}, "HTTPStatusCode": 405 } + }, + "get-acl-delete-marker": { + "Error": { + "Code": "NoSuchKey", + "Key": "test-key-versioned", + "Message": "The specified key does not exist." + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 404 + } + }, + "put-acl-delete-marker-version-id": { + "Error": { + "Code": "MethodNotAllowed", + "Message": "The specified method is not allowed against this resource.", + "Method": "PUT", + "ResourceType": "DeleteMarker" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 405 + } + }, + "get-acl-delete-marker-version-id": { + "Error": { + "Code": "MethodNotAllowed", + "Message": "The specified method is not allowed against this resource.", + "Method": "GET", + "ResourceType": "DeleteMarker" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 405 + } } } }, diff --git a/tests/unit/test_s3.py b/tests/unit/test_s3.py index 33a558f1c1902..021a69e8049c1 100644 --- a/tests/unit/test_s3.py +++ b/tests/unit/test_s3.py @@ -9,6 +9,7 @@ from requests.models import Response from localstack.aws.api import RequestContext +from localstack.aws.api.s3 import InvalidArgument from localstack.constants import LOCALHOST, S3_VIRTUAL_HOSTNAME from localstack.http import Request from localstack.services.infra import patch_instance_tracker_meta @@ -19,6 +20,7 @@ from localstack.services.s3.legacy import multipart_content, s3_listener, s3_starter, s3_utils from localstack.services.s3.v3.models import S3Multipart, S3Object, S3Part from localstack.services.s3.v3.storage.ephemeral import EphemeralS3ObjectStore +from localstack.services.s3.validation import validate_canned_acl from localstack.utils.strings import short_uid @@ -534,38 +536,51 @@ def test_is_valid_canonical_id(self): for canonical_id, expected_result in canonical_ids: assert s3_utils_asf.is_valid_canonical_id(canonical_id) == expected_result - def test_get_header_name(self): + @pytest.mark.parametrize( + "request_member, permission, response_header", + [ + ("GrantFullControl", "FULL_CONTROL", "x-amz-grant-full-control"), + ("GrantRead", "READ", "x-amz-grant-read"), + ("GrantReadACP", "READ_ACP", "x-amz-grant-read-acp"), + ("GrantWrite", "WRITE", "x-amz-grant-write"), + ("GrantWriteACP", "WRITE_ACP", "x-amz-grant-write-acp"), + ], + ) + def test_get_permission_from_request_header_to_response_header( + self, request_member, permission, response_header + ): """ Test to transform shape member names into their header location We could maybe use the specs for this """ - query_params = [ - ("GrantFullControl", "x-amz-grant-full-control"), - ("GrantRead", "x-amz-grant-read"), - ("GrantReadACP", "x-amz-grant-read-acp"), - ("GrantWrite", "x-amz-grant-write"), - ("GrantWriteACP", "x-amz-grant-write-acp"), - ] + parsed_permission = s3_utils_asf.get_permission_from_header(request_member) + assert parsed_permission == permission + assert s3_utils_asf.get_permission_header_name(parsed_permission) == response_header - for query_param, expected_header_name in query_params: - assert s3_utils_asf.get_header_name(query_param) == expected_header_name - - def test_is_canned_acl_valid(self): - canned_acls = [ - ("private", True), - ("public-read", True), - ("public-read-write", True), - ("authenticated-read", True), - ("aws-exec-read", True), - ("bucket-owner-read", True), - ("bucket-owner-full-control", True), - ("not-a-canned-one", False), - ("aws--exec-read", False), - ("log-delivery-write", True), - ] - - for canned_acl, expected_result in canned_acls: - assert s3_utils_asf.is_canned_acl_bucket_valid(canned_acl) == expected_result + @pytest.mark.parametrize( + "canned_acl, raise_exception", + [ + ("private", False), + ("public-read", False), + ("public-read-write", False), + ("authenticated-read", False), + ("aws-exec-read", False), + ("bucket-owner-read", False), + ("bucket-owner-full-control", False), + ("not-a-canned-one", True), + ("aws--exec-read", True), + ("log-delivery-write", False), + ], + ) + def test_validate_canned_acl(self, canned_acl, raise_exception): + if raise_exception: + with pytest.raises(InvalidArgument) as e: + validate_canned_acl(canned_acl) + assert e.value.ArgumentName == "x-amz-acl" + assert e.value.ArgumentValue == canned_acl + + else: + validate_canned_acl(canned_acl) def test_s3_bucket_name(self): bucket_names = [ From 2f79dec6f9e5afad341530d197892725845589dc Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Tue, 15 Aug 2023 23:50:18 +0200 Subject: [PATCH 2/3] implement Get/PutObjectACL tests --- tests/aws/services/s3/test_s3.py | 207 +++++++++++++- tests/aws/services/s3/test_s3.snapshot.json | 284 ++++++++++++++++++++ 2 files changed, 490 insertions(+), 1 deletion(-) diff --git a/tests/aws/services/s3/test_s3.py b/tests/aws/services/s3/test_s3.py index 4b2a8174a751f..62d8d3046b1dc 100644 --- a/tests/aws/services/s3/test_s3.py +++ b/tests/aws/services/s3/test_s3.py @@ -2211,6 +2211,211 @@ def test_s3_bucket_acl_exceptions(self, s3_bucket, snapshot, aws_client): snapshot.match("put-bucket-two-type-acl-acp", e.value.response) + @markers.aws.validated + @markers.snapshot.skip_snapshot_verify( + condition=is_old_provider, paths=["$..Grants..Grantee.DisplayName", "$..Grants..Grantee.ID"] + ) + @markers.snapshot.skip_snapshot_verify( + condition=lambda: not is_native_provider(), + paths=["$..ServerSideEncryption"], + ) + def test_s3_object_acl(self, s3_bucket, allow_bucket_acl, snapshot, aws_client): + # loosely based on + # https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html + snapshot.add_transformer( + [ + snapshot.transform.key_value("DisplayName"), + snapshot.transform.key_value("ID", value_replacement="owner-id"), + ] + ) + list_bucket_output = aws_client.s3.list_buckets() + owner = list_bucket_output["Owner"] + object_key = "object-key-acl" + put_object = aws_client.s3.put_object(Bucket=s3_bucket, Key=object_key) + snapshot.match("put-object-default-acl", put_object) + + response = aws_client.s3.get_object_acl(Bucket=s3_bucket, Key=object_key) + snapshot.match("get-object-acl-default", response) + + put_object_acl = aws_client.s3.put_object_acl( + Bucket=s3_bucket, Key=object_key, ACL="public-read" + ) + snapshot.match("put-object-acl", put_object_acl) + + response = aws_client.s3.get_object_acl(Bucket=s3_bucket, Key=object_key) + snapshot.match("get-object-acl", response) + + # this a bucket URI? + aws_client.s3.put_object_acl( + Bucket=s3_bucket, + Key=object_key, + GrantRead='uri="http://acs.amazonaws.com/groups/s3/LogDelivery"', + ) + + response = aws_client.s3.get_object_acl(Bucket=s3_bucket, Key=object_key) + snapshot.match("get-object-grant-acl", response) + + # Owner is mandatory, otherwise raise MalformedXML + acp = { + "Owner": owner, + "Grants": [ + { + "Grantee": {"ID": owner["ID"], "Type": "CanonicalUser"}, + "Permission": "FULL_CONTROL", + }, + { + "Grantee": { + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", + "Type": "Group", + }, + "Permission": "WRITE", + }, + ], + } + aws_client.s3.put_object_acl(Bucket=s3_bucket, Key=object_key, AccessControlPolicy=acp) + + response = aws_client.s3.get_object_acl(Bucket=s3_bucket, Key=object_key) + snapshot.match("get-object-acp-acl", response) + + @markers.aws.validated + @pytest.mark.skipif(LEGACY_S3_PROVIDER, reason="Behaviour not implemented in legacy provider") + @pytest.mark.xfail( + condition=not config.NATIVE_S3_PROVIDER, + reason="Behaviour is not in line with AWS, does not validate properly", + ) + def test_s3_object_acl_exceptions(self, s3_bucket, snapshot, aws_client): + list_bucket_output = aws_client.s3.list_buckets() + owner = list_bucket_output["Owner"] + object_key = "object-key-acl" + aws_client.s3.put_object(Bucket=s3_bucket, Key=object_key) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object(Bucket=s3_bucket, Key=object_key, ACL="fake-acl") + snapshot.match("put-object-canned-acl", e.value.response) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl(Bucket=s3_bucket, Key=object_key, ACL="fake-acl") + snapshot.match("put-object-acl-canned-acl", e.value.response) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl( + Bucket=s3_bucket, + Key=object_key, + GrantWrite='uri="http://acs.amazonaws.com/groups/s3/FakeGroup"', + ) + snapshot.match("put-object-grant-acl-fake-uri", e.value.response) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl( + Bucket=s3_bucket, Key=object_key, GrantWrite='fakekey="1234"' + ) + snapshot.match("put-object-grant-acl-fake-key", e.value.response) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl( + Bucket=s3_bucket, Key=object_key, GrantWrite='id="wrong-id"' + ) + + snapshot.match("put-object-grant-acl-wrong-id", e.value.response) + + acp = { + "Grants": [ + { + "Grantee": { + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", + "Type": "Group", + }, + "Permission": "WRITE", + } + ] + } + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl(Bucket=s3_bucket, Key=object_key, AccessControlPolicy=acp) + snapshot.match("put-object-acp-acl-1", e.value.response) + + # add Owner, but modify the permission + acp["Owner"] = owner + acp["Grants"][0]["Permission"] = "WRONG-PERMISSION" + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl(Bucket=s3_bucket, Key=object_key, AccessControlPolicy=acp) + snapshot.match("put-object-acp-acl-2", e.value.response) + + # restore good permission, but put bad format Owner ID + acp["Owner"] = {"ID": "wrong-id"} + acp["Grants"][0]["Permission"] = "FULL_CONTROL" + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl(Bucket=s3_bucket, Key=object_key, AccessControlPolicy=acp) + snapshot.match("put-object-acp-acl-3", e.value.response) + + # restore owner, but wrong URI + acp["Owner"] = owner + acp["Grants"][0]["Grantee"]["URI"] = "http://acs.amazonaws.com/groups/s3/FakeGroup" + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl(Bucket=s3_bucket, Key=object_key, AccessControlPolicy=acp) + snapshot.match("put-object-acp-acl-4", e.value.response) + + # different type of failing grantee (CanonicalUser/ID) + acp["Grants"][0]["Grantee"]["Type"] = "CanonicalUser" + acp["Grants"][0]["Grantee"]["ID"] = "wrong-id" + acp["Grants"][0]["Grantee"].pop("URI") + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl(Bucket=s3_bucket, Key=object_key, AccessControlPolicy=acp) + snapshot.match("put-object-acp-acl-5", e.value.response) + + # different type of failing grantee (Wrong type) + acp["Grants"][0]["Grantee"]["Type"] = "BadType" + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl(Bucket=s3_bucket, Key=object_key, AccessControlPolicy=acp) + snapshot.match("put-object-acp-acl-6", e.value.response) + + # test setting empty ACP + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl(Bucket=s3_bucket, Key=object_key, AccessControlPolicy={}) + + snapshot.match("put-object-empty-acp", e.value.response) + + # test setting nothing + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl(Bucket=s3_bucket, Key=object_key) + + snapshot.match("put-object-acl-empty", e.value.response) + + # test setting two different kind of valid ACL + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl( + Bucket=s3_bucket, + Key=object_key, + ACL="private", + GrantRead='uri="http://acs.amazonaws.com/groups/s3/LogDelivery"', + ) + + snapshot.match("put-object-two-type-acl", e.value.response) + + # test setting again two different kind of valid ACL + acp = { + "Owner": owner, + "Grants": [ + { + "Grantee": {"ID": owner["ID"], "Type": "CanonicalUser"}, + "Permission": "FULL_CONTROL", + }, + ], + } + with pytest.raises(ClientError) as e: + aws_client.s3.put_object_acl( + Bucket=s3_bucket, + Key=object_key, + ACL="private", + AccessControlPolicy=acp, + ) + + snapshot.match("put-object-two-type-acl-acp", e.value.response) + @markers.aws.validated @markers.snapshot.skip_snapshot_verify(paths=["$..Restore"]) @markers.snapshot.skip_snapshot_verify( @@ -2523,7 +2728,7 @@ def test_put_object_chunked_newlines(self, s3_bucket, aws_client): @markers.aws.only_localstack @pytest.mark.xfail( reason="Not implemented in other providers than stream", - condition=not STREAM_S3_PROVIDER, + condition=not STREAM_S3_PROVIDER or not NATIVE_S3_PROVIDER, ) def test_put_object_chunked_newlines_with_checksum(self, s3_bucket, aws_client): # Boto still does not support chunk encoding, which means we can't test with the client nor diff --git a/tests/aws/services/s3/test_s3.snapshot.json b/tests/aws/services/s3/test_s3.snapshot.json index 10db14faf0b0e..7da5fd1752139 100644 --- a/tests/aws/services/s3/test_s3.snapshot.json +++ b/tests/aws/services/s3/test_s3.snapshot.json @@ -9822,5 +9822,289 @@ } } } + }, + "tests/aws/s3/test_s3.py::TestS3::test_s3_object_acl": { + "recorded-date": "15-08-2023, 23:41:05", + "recorded-content": { + "put-object-default-acl": { + "ETag": "\"d41d8cd98f00b204e9800998ecf8427e\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-object-acl-default": { + "Grants": [ + { + "Grantee": { + "DisplayName": "", + "ID": "", + "Type": "CanonicalUser" + }, + "Permission": "FULL_CONTROL" + } + ], + "Owner": { + "DisplayName": "", + "ID": "" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "put-object-acl": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-object-acl": { + "Grants": [ + { + "Grantee": { + "DisplayName": "", + "ID": "", + "Type": "CanonicalUser" + }, + "Permission": "FULL_CONTROL" + }, + { + "Grantee": { + "Type": "Group", + "URI": "http://acs.amazonaws.com/groups/global/AllUsers" + }, + "Permission": "READ" + } + ], + "Owner": { + "DisplayName": "", + "ID": "" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-object-grant-acl": { + "Grants": [ + { + "Grantee": { + "Type": "Group", + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery" + }, + "Permission": "READ" + } + ], + "Owner": { + "DisplayName": "", + "ID": "" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-object-acp-acl": { + "Grants": [ + { + "Grantee": { + "DisplayName": "", + "ID": "", + "Type": "CanonicalUser" + }, + "Permission": "FULL_CONTROL" + }, + { + "Grantee": { + "Type": "Group", + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery" + }, + "Permission": "WRITE" + } + ], + "Owner": { + "DisplayName": "", + "ID": "" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } + }, + "tests/aws/s3/test_s3.py::TestS3::test_s3_object_acl_exceptions": { + "recorded-date": "15-08-2023, 23:47:00", + "recorded-content": { + "put-object-canned-acl": { + "Error": { + "ArgumentName": "x-amz-acl", + "ArgumentValue": "fake-acl", + "Code": "InvalidArgument", + "Message": null + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-acl-canned-acl": { + "Error": { + "ArgumentName": "x-amz-acl", + "ArgumentValue": "fake-acl", + "Code": "InvalidArgument", + "Message": null + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-grant-acl-fake-uri": { + "Error": { + "ArgumentName": "uri", + "ArgumentValue": "http://acs.amazonaws.com/groups/s3/FakeGroup", + "Code": "InvalidArgument", + "Message": "Invalid group uri" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-grant-acl-fake-key": { + "Error": { + "ArgumentName": "x-amz-grant-write", + "ArgumentValue": "fakekey=\"1234\"", + "Code": "InvalidArgument", + "Message": "Argument format not recognized" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-grant-acl-wrong-id": { + "Error": { + "ArgumentName": "id", + "ArgumentValue": "wrong-id", + "Code": "InvalidArgument", + "Message": "Invalid id" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-acp-acl-1": { + "Error": { + "Code": "MalformedACLError", + "Message": "The XML you provided was not well-formed or did not validate against our published schema" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-acp-acl-2": { + "Error": { + "Code": "MalformedACLError", + "Message": "The XML you provided was not well-formed or did not validate against our published schema" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-acp-acl-3": { + "Error": { + "ArgumentName": "CanonicalUser/ID", + "ArgumentValue": "wrong-id", + "Code": "InvalidArgument", + "Message": "Invalid id" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-acp-acl-4": { + "Error": { + "ArgumentName": "Group/URI", + "ArgumentValue": "http://acs.amazonaws.com/groups/s3/FakeGroup", + "Code": "InvalidArgument", + "Message": "Invalid group uri" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-acp-acl-5": { + "Error": { + "ArgumentName": "CanonicalUser/ID", + "ArgumentValue": "wrong-id", + "Code": "InvalidArgument", + "Message": "Invalid id" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-acp-acl-6": { + "Error": { + "Code": "MalformedACLError", + "Message": "The XML you provided was not well-formed or did not validate against our published schema" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-empty-acp": { + "Error": { + "Code": "MalformedACLError", + "Message": "The XML you provided was not well-formed or did not validate against our published schema" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-acl-empty": { + "Error": { + "Code": "MissingSecurityHeader", + "Message": "Your request was missing a required header", + "MissingHeaderName": "x-amz-acl" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-two-type-acl": { + "Error": { + "Code": "InvalidRequest", + "Message": "Specifying both Canned ACLs and Header Grants is not allowed" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "put-object-two-type-acl-acp": { + "Error": { + "Code": "UnexpectedContent", + "Message": "This request does not support content" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + } + } } } From 5b821fd89255297920d264922a6a7eb5db9a4307 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Mon, 21 Aug 2023 15:56:43 +0200 Subject: [PATCH 3/3] modify snapshot path --- tests/aws/services/s3/test_s3.snapshot.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/aws/services/s3/test_s3.snapshot.json b/tests/aws/services/s3/test_s3.snapshot.json index 7da5fd1752139..5331420fdebdb 100644 --- a/tests/aws/services/s3/test_s3.snapshot.json +++ b/tests/aws/services/s3/test_s3.snapshot.json @@ -9823,7 +9823,7 @@ } } }, - "tests/aws/s3/test_s3.py::TestS3::test_s3_object_acl": { + "tests/aws/services/s3/test_s3.py::TestS3::test_s3_object_acl": { "recorded-date": "15-08-2023, 23:41:05", "recorded-content": { "put-object-default-acl": { @@ -9935,7 +9935,7 @@ } } }, - "tests/aws/s3/test_s3.py::TestS3::test_s3_object_acl_exceptions": { + "tests/aws/services/s3/test_s3.py::TestS3::test_s3_object_acl_exceptions": { "recorded-date": "15-08-2023, 23:47:00", "recorded-content": { "put-object-canned-acl": {