Thanks to visit codestin.com
Credit goes to github.com

Skip to content

implement S3 native ACL #8902

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 21, 2023
Merged

implement S3 native ACL #8902

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
04160b8
implement ACL
bentsku Aug 13, 2023
2f79dec
implement Get/PutObjectACL tests
bentsku Aug 15, 2023
5b821fd
modify snapshot path
bentsku Aug 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 12 additions & 17 deletions localstack/services/s3/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
from localstack.aws.api.s3 import (
BucketCannedACL,
ObjectCannedACL,
Grantee,
Permission,
PublicAccessBlockConfiguration,
ServerSideEncryption,
ServerSideEncryptionByDefault,
ServerSideEncryptionRule,
StorageClass,
)
from localstack.aws.api.s3 import Type as GranteeType

S3_VIRTUAL_HOST_FORWARDED_HEADER = "x-s3-vhost-forwarded-for"

Expand All @@ -16,23 +16,14 @@
This is minimum size allowed by S3 when uploading more than one part for a Multipart Upload, except for the last part
"""

VALID_CANNED_ACLS_BUCKET = {
# https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
# bucket-owner-read + bucket-owner-full-control are allowed, but ignored for buckets
ObjectCannedACL.private,
ObjectCannedACL.authenticated_read,
ObjectCannedACL.aws_exec_read,
ObjectCannedACL.bucket_owner_full_control,
ObjectCannedACL.bucket_owner_read,
ObjectCannedACL.public_read,
ObjectCannedACL.public_read_write,
BucketCannedACL.log_delivery_write,
}
AUTHENTICATED_USERS_ACL_GROUP = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
ALL_USERS_ACL_GROUP = "http://acs.amazonaws.com/groups/global/AllUsers"
LOG_DELIVERY_ACL_GROUP = "http://acs.amazonaws.com/groups/s3/LogDelivery"

VALID_ACL_PREDEFINED_GROUPS = {
"http://acs.amazonaws.com/groups/global/AuthenticatedUsers",
"http://acs.amazonaws.com/groups/global/AllUsers",
"http://acs.amazonaws.com/groups/s3/LogDelivery",
AUTHENTICATED_USERS_ACL_GROUP,
ALL_USERS_ACL_GROUP,
LOG_DELIVERY_ACL_GROUP,
}

VALID_GRANTEE_PERMISSIONS = {
Expand Down Expand Up @@ -117,3 +108,7 @@
RestrictPublicBuckets=True,
IgnorePublicAcls=True,
)

AUTHENTICATED_USERS_ACL_GRANTEE = Grantee(URI=AUTHENTICATED_USERS_ACL_GROUP, Type=GranteeType.Group)
ALL_USERS_ACL_GRANTEE = Grantee(URI=ALL_USERS_ACL_GROUP, Type=GranteeType.Group)
LOG_DELIVERY_ACL_GRANTEE = Grantee(URI=LOG_DELIVERY_ACL_GROUP, Type=GranteeType.Group)
12 changes: 8 additions & 4 deletions localstack/services/s3/provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,18 +170,19 @@
get_key_from_moto_bucket,
get_lifecycle_rule_from_object,
get_object_checksum_for_algorithm,
get_permission_from_header,
is_key_expired,
serialize_expiration_header,
validate_kms_key_id,
verify_checksum,
)
from localstack.services.s3.validation import (
parse_grants_in_headers,
validate_acl_acp,
validate_bucket_analytics_configuration,
validate_bucket_intelligent_tiering_configuration,
validate_bucket_name,
validate_canned_acl,
validate_grantee_in_headers,
validate_inventory_configuration,
validate_lifecycle_configuration,
validate_website_configuration,
Expand Down Expand Up @@ -1183,8 +1184,10 @@ def put_bucket_acl(
validate_canned_acl(canned_acl)

elif present_headers:
for key, grantees_values in present_headers:
validate_grantee_in_headers(key, grantees_values)
for key in grant_keys:
if grantees_values := request.get(key, ""): # noqa
permission = get_permission_from_header(key)
parse_grants_in_headers(permission, grantees_values)

elif acp := request.get("AccessControlPolicy"):
validate_acl_acp(acp)
Expand Down Expand Up @@ -1227,7 +1230,8 @@ def put_object_acl(
]
for key in grant_keys:
if grantees_values := request.get(key, ""): # noqa
validate_grantee_in_headers(key, grantees_values)
permission = get_permission_from_header(key)
parse_grants_in_headers(permission, grantees_values)

if acp := request.get("AccessControlPolicy"):
validate_acl_acp(acp)
Expand Down
60 changes: 54 additions & 6 deletions localstack/services/s3/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,16 @@
from localstack import config
from localstack.aws.api import CommonServiceException, RequestContext
from localstack.aws.api.s3 import (
AccessControlPolicy,
BucketCannedACL,
BucketName,
ChecksumAlgorithm,
CopyObjectRequest,
CopySource,
ETag,
GetObjectRequest,
Grant,
Grantee,
HeadObjectRequest,
InvalidArgument,
InvalidRange,
Expand All @@ -34,23 +38,28 @@
MethodNotAllowed,
NoSuchBucket,
NoSuchKey,
ObjectCannedACL,
ObjectKey,
ObjectSize,
ObjectVersionId,
Owner,
Permission,
PreconditionFailed,
SSEKMSKeyId,
TaggingHeader,
TagSet,
)
from localstack.aws.api.s3 import Type as GranteeType
from localstack.aws.connect import connect_to
from localstack.services.s3.constants import (
ALL_USERS_ACL_GRANTEE,
AUTHENTICATED_USERS_ACL_GRANTEE,
LOG_DELIVERY_ACL_GRANTEE,
S3_CHUNK_SIZE,
S3_VIRTUAL_HOST_FORWARDED_HEADER,
SIGNATURE_V2_PARAMS,
SIGNATURE_V4_PARAMS,
SYSTEM_METADATA_SETTABLE_HEADERS,
VALID_CANNED_ACLS_BUCKET,
)
from localstack.services.s3.exceptions import InvalidRequest
from localstack.utils.aws import arns
Expand Down Expand Up @@ -325,13 +334,13 @@ def is_bucket_name_valid(bucket_name: str) -> bool:
return True if re.match(BUCKET_NAME_REGEX, bucket_name) else False


def is_canned_acl_bucket_valid(canned_acl: str) -> bool:
return canned_acl in VALID_CANNED_ACLS_BUCKET
def get_permission_header_name(permission: Permission) -> str:
return f"x-amz-grant-{permission.replace('_', '-').lower()}"


def get_header_name(capitalized_field: str) -> str:
headers_parts = re.split(r"([A-Z][a-z]+)", capitalized_field)
return f"x-amz-{'-'.join([part.lower() for part in headers_parts if part])}"
def get_permission_from_header(capitalized_field: str) -> Permission:
headers_parts = [part.upper() for part in re.split(r"([A-Z][a-z]+)", capitalized_field) if part]
return "_".join(headers_parts[1:])


def is_valid_canonical_id(canonical_id: str) -> bool:
Expand Down Expand Up @@ -860,3 +869,42 @@ def validate_failed_precondition(
code="NotModified",
status_code=304,
)


def get_canned_acl(
canned_acl: BucketCannedACL | ObjectCannedACL, owner: Owner
) -> AccessControlPolicy:
"""
Return the proper Owner and Grants from a CannedACL
See https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
:param canned_acl: an S3 CannedACL
:param owner: the current owner of the bucket or object
:return: an AccessControlPolicy containing the Grants and Owner
"""
owner_grantee = Grantee(**owner, Type=GranteeType.CanonicalUser)
grants = [Grant(Grantee=owner_grantee, Permission=Permission.FULL_CONTROL)]

match canned_acl:
case ObjectCannedACL.private:
pass # no other permissions
case ObjectCannedACL.public_read:
grants.append(Grant(Grantee=ALL_USERS_ACL_GRANTEE, Permission=Permission.READ))

case ObjectCannedACL.public_read_write:
grants.append(Grant(Grantee=ALL_USERS_ACL_GRANTEE, Permission=Permission.READ))
grants.append(Grant(Grantee=ALL_USERS_ACL_GRANTEE, Permission=Permission.WRITE))
case ObjectCannedACL.authenticated_read:
grants.append(
Grant(Grantee=AUTHENTICATED_USERS_ACL_GRANTEE, Permission=Permission.READ)
)
case ObjectCannedACL.bucket_owner_read:
pass # TODO: bucket owner ACL
case ObjectCannedACL.bucket_owner_full_control:
pass # TODO: bucket owner ACL
case ObjectCannedACL.aws_exec_read:
pass # TODO: bucket owner, EC2 Read
case BucketCannedACL.log_delivery_write:
grants.append(Grant(Grantee=LOG_DELIVERY_ACL_GRANTEE, Permission=Permission.READ_ACP))
grants.append(Grant(Grantee=LOG_DELIVERY_ACL_GRANTEE, Permission=Permission.WRITE))

return AccessControlPolicy(Owner=owner, Grants=grants)
32 changes: 24 additions & 8 deletions localstack/services/s3/v3/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from localstack import config
from localstack.aws.api import CommonServiceException
from localstack.aws.api.s3 import (
AccessControlPolicy,
AccountId,
AnalyticsConfiguration,
AnalyticsId,
Expand Down Expand Up @@ -67,7 +68,6 @@
S3_UPLOAD_PART_MIN_SIZE,
)
from localstack.services.s3.utils import (
get_owner_for_account_id,
iso_8601_datetime_without_milliseconds_s3,
rfc_1123_datetime,
)
Expand Down Expand Up @@ -101,7 +101,7 @@ class S3Bucket:
lifecycle_rules: Optional[LifecycleRules]
policy: Optional[Policy]
website_configuration: Optional[WebsiteConfiguration]
acl: str # TODO: change this
acl: AccessControlPolicy
cors_rules: Optional[CORSConfiguration]
logging: LoggingEnabled
notification_configuration: NotificationConfiguration
Expand All @@ -124,7 +124,8 @@ def __init__(
name: BucketName,
account_id: AccountId,
bucket_region: BucketRegion,
acl=None, # TODO: validate ACL first, create utils for validating and consolidating
owner: Owner,
acl: AccessControlPolicy = None,
object_ownership: ObjectOwnership = None,
object_lock_enabled_for_bucket: bool = None,
):
Expand Down Expand Up @@ -153,9 +154,9 @@ def __init__(
self.inventory_configurations = {}
self.object_lock_default_retention = {}
self.replication = None

self.acl = acl
# see https://docs.aws.amazon.com/AmazonS3/latest/API/API_Owner.html
self.owner = get_owner_for_account_id(account_id)
self.owner = owner
self.bucket_arn = arns.s3_bucket_arn(self.name)

def get_object(
Expand Down Expand Up @@ -225,6 +226,16 @@ def get_object(
raise NoSuchKey("The specified key does not exist.", Key=key)

elif raise_for_delete_marker and isinstance(s3_object, S3DeleteMarker):
if http_method not in ("HEAD", "GET"):
raise MethodNotAllowed(
"The specified method is not allowed against this resource.",
Method=http_method,
ResourceType="DeleteMarker",
DeleteMarker=True,
Allow="DELETE",
VersionId=s3_object.version_id,
)

raise NoSuchKey(
"The specified key does not exist.",
Key=key,
Expand All @@ -240,6 +251,7 @@ class S3Object:
key: ObjectKey
version_id: Optional[ObjectVersionId]
bucket: BucketName
owner: Optional[Owner]
size: Optional[Size]
etag: Optional[ETag]
user_metadata: Metadata
Expand All @@ -257,7 +269,7 @@ class S3Object:
lock_legal_status: Optional[ObjectLockLegalHoldStatus]
lock_until: Optional[datetime]
website_redirect_location: Optional[WebsiteRedirectLocation]
acl: Optional[str] # TODO: we need to change something here, how it's done?
acl: Optional[AccessControlPolicy]
is_current: bool
parts: Optional[dict[int, tuple[int, int]]]
restore: Optional[Restore]
Expand All @@ -282,7 +294,8 @@ def __init__(
lock_legal_status: Optional[ObjectLockLegalHoldStatus] = None,
lock_until: Optional[datetime] = None,
website_redirect_location: Optional[WebsiteRedirectLocation] = None,
acl: Optional[str] = None, # TODO
acl: Optional[AccessControlPolicy] = None, # TODO
owner: Optional[Owner] = None,
):
self.key = key
self.user_metadata = (
Expand All @@ -309,6 +322,7 @@ def __init__(
self.last_modified = datetime.now(tz=_gmt_zone_info)
self.parts = {}
self.restore = None
self.owner = owner

def get_system_metadata_fields(self) -> dict:
headers = {
Expand Down Expand Up @@ -433,11 +447,12 @@ def __init__(
lock_legal_status: Optional[ObjectLockLegalHoldStatus] = None,
lock_until: Optional[datetime] = None,
website_redirect_location: Optional[WebsiteRedirectLocation] = None,
acl: Optional[str] = None, # TODO
acl: Optional[AccessControlPolicy] = None, # TODO
user_metadata: Optional[Metadata] = None,
system_metadata: Optional[Metadata] = None,
initiator: Optional[Owner] = None,
tagging: Optional[dict[str, str]] = None,
owner: Optional[Owner] = None,
):
self.id = token_urlsafe(96) # MultipartUploadId is 128 characters long
self.initiated = datetime.now(tz=_gmt_zone_info)
Expand All @@ -461,6 +476,7 @@ def __init__(
lock_until=lock_until,
website_redirect_location=website_redirect_location,
acl=acl,
owner=owner,
)

def complete_multipart(self, parts: CompletedPartList):
Expand Down
Loading