diff --git a/localstack/aws/api/s3/__init__.py b/localstack/aws/api/s3/__init__.py index 179f86ca8e580..3ddb3332989d6 100644 --- a/localstack/aws/api/s3/__init__.py +++ b/localstack/aws/api/s3/__init__.py @@ -652,6 +652,7 @@ class InvalidArgument(ServiceException): status_code: int = 400 ArgumentName: Optional[ArgumentName] ArgumentValue: Optional[ArgumentValue] + HostId: Optional[HostId] class SignatureDoesNotMatch(ServiceException): @@ -3011,6 +3012,34 @@ class DeleteResult(TypedDict, total=False): Errors: Optional[Errors] +class PostObjectRequest(ServiceRequest): + Body: Optional[IO[Body]] + Bucket: BucketName + + +class PostResponse(TypedDict, total=False): + StatusCode: Optional[GetObjectResponseStatusCode] + Location: Optional[Location] + LocationHeader: Optional[Location] + Bucket: Optional[BucketName] + Key: Optional[ObjectKey] + Expiration: Optional[Expiration] + ETag: Optional[ETag] + ETagHeader: Optional[ETag] + ChecksumCRC32: Optional[ChecksumCRC32] + ChecksumCRC32C: Optional[ChecksumCRC32C] + ChecksumSHA1: Optional[ChecksumSHA1] + ChecksumSHA256: Optional[ChecksumSHA256] + ServerSideEncryption: Optional[ServerSideEncryption] + VersionId: Optional[ObjectVersionId] + SSECustomerAlgorithm: Optional[SSECustomerAlgorithm] + SSECustomerKeyMD5: Optional[SSECustomerKeyMD5] + SSEKMSKeyId: Optional[SSEKMSKeyId] + SSEKMSEncryptionContext: Optional[SSEKMSEncryptionContext] + BucketKeyEnabled: Optional[BucketKeyEnabled] + RequestCharged: Optional[RequestCharged] + + class S3Api: service = "s3" @@ -4202,3 +4231,9 @@ def write_get_object_response( bucket_key_enabled: BucketKeyEnabled = None, ) -> None: raise NotImplementedError + + @handler("PostObject") + def post_object( + self, context: RequestContext, bucket: BucketName, body: IO[Body] = None + ) -> PostResponse: + raise NotImplementedError diff --git a/localstack/aws/protocol/serializer.py b/localstack/aws/protocol/serializer.py index 1154e2099a393..687dba4270047 100644 --- a/localstack/aws/protocol/serializer.py +++ b/localstack/aws/protocol/serializer.py @@ -1355,6 +1355,27 @@ class S3ResponseSerializer(RestXMLResponseSerializer): SUPPORTED_MIME_TYPES = [TEXT_XML, APPLICATION_XML] + def _serialize_response( + self, + parameters: dict, + response: HttpResponse, + shape: Optional[Shape], + shape_members: dict, + operation_model: OperationModel, + mime_type: str, + ) -> None: + header_params, payload_params = self._partition_members(parameters, shape) + self._process_header_members(header_params, response, shape) + # "HEAD" responses are basically "GET" responses without the actual body. + # Do not process the body payload in this case (setting a body could also manipulate the headers) + # If the response is a redirection, the body should be empty as well + if operation_model.http.get("method") != "HEAD" and not 300 <= response.status_code < 400: + self._serialize_payload( + payload_params, response, shape, shape_members, operation_model, mime_type + ) + self._serialize_content_type(response, shape, shape_members, mime_type) + self._prepare_additional_traits_in_response(response, operation_model) + def _serialize_error( self, error: ServiceException, @@ -1381,11 +1402,9 @@ def _prepare_additional_traits_in_response( ): """Adds the request ID to the headers (in contrast to the body - as in the Query protocol).""" response = super()._prepare_additional_traits_in_response(response, operation_model) - request_id = gen_amzn_requestid_long() - response.headers["x-amz-request-id"] = request_id response.headers[ "x-amz-id-2" - ] = f"MzRISOwyjmnup{request_id}7/JypPGXLh0OVFGcJaaO3KW/hRAqKOpIEEp" + ] = f"MzRISOwyjmnup{response.headers['x-amz-request-id']}7/JypPGXLh0OVFGcJaaO3KW/hRAqKOpIEEp" return response def _add_error_tags( diff --git a/localstack/aws/spec-patches.json b/localstack/aws/spec-patches.json index 947500c3dae92..3fb2b6d6c9eda 100644 --- a/localstack/aws/spec-patches.json +++ b/localstack/aws/spec-patches.json @@ -267,6 +267,9 @@ }, "ArgumentValue": { "shape": "ArgumentValue" + }, + "HostId": { + "shape": "HostId" } }, "error": { @@ -439,6 +442,163 @@ "exception": true } }, + { + "op": "add", + "path": "/operations/PostObject", + "value": { + "name":"PostObject", + "http":{ + "method":"POST", + "requestUri":"/{Bucket}" + }, + "input":{"shape":"PostObjectRequest"}, + "output":{"shape":"PostResponse"}, + "documentationUrl":"http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html", + "documentation":"

The POST operation adds an object to a specified bucket by using HTML forms. POST is an alternate form of PUT that enables browser-based uploads as a way of putting objects in buckets. Parameters that are passed to PUT through HTTP Headers are instead passed as form fields to POST in the multipart/form-data encoded message body. To add an object to a bucket, you must have WRITE access on the bucket. Amazon S3 never stores partial objects. If you receive a successful response, you can be confident that the entire object was stored.

" + } + }, + { + "op": "add", + "path": "/shapes/PostObjectRequest", + "value": { + "type":"structure", + "required":[ + "Bucket" + ], + "members":{ + "Body":{ + "shape":"Body", + "documentation":"

Object data.

", + "streaming":true + }, + "Bucket":{ + "shape":"BucketName", + "documentation":"

The bucket name to which the PUT action was initiated.

When using this action with an access point, you must direct requests to the access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. When using this action with an access point through the Amazon Web Services SDKs, you provide the access point ARN in place of the bucket name. For more information about access point ARNs, see Using access points in the Amazon S3 User Guide.

When using this action with Amazon S3 on Outposts, you must direct requests to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form AccessPointName-AccountId.outpostID.s3-outposts.Region.amazonaws.com. When using this action with S3 on Outposts through the Amazon Web Services SDKs, you provide the Outposts bucket ARN in place of the bucket name. For more information about S3 on Outposts ARNs, see Using Amazon S3 on Outposts in the Amazon S3 User Guide.

", + "location":"uri", + "locationName":"Bucket" + } + }, + "payload":"Body" + } + }, + { + "op": "add", + "path": "/shapes/PostResponse", + "value": { + "type":"structure", + "members":{ + "StatusCode": { + "shape": "GetObjectResponseStatusCode", + "location": "statusCode" + }, + "Location":{ + "shape":"Location", + "documentation":"

The URI that identifies the newly created object.

" + }, + "LocationHeader":{ + "shape":"Location", + "documentation":"

The URI that identifies the newly created object.

", + "location": "header", + "locationName": "Location" + }, + "Bucket":{ + "shape":"BucketName", + "documentation":"

The name of the bucket that contains the newly created object. Does not return the access point ARN or access point alias if used.

When using this action with an access point, you must direct requests to the access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. When using this action with an access point through the Amazon Web Services SDKs, you provide the access point ARN in place of the bucket name. For more information about access point ARNs, see Using access points in the Amazon S3 User Guide.

When using this action with Amazon S3 on Outposts, you must direct requests to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form AccessPointName-AccountId.outpostID.s3-outposts.Region.amazonaws.com. When using this action with S3 on Outposts through the Amazon Web Services SDKs, you provide the Outposts bucket ARN in place of the bucket name. For more information about S3 on Outposts ARNs, see Using Amazon S3 on Outposts in the Amazon S3 User Guide.

" + }, + "Key":{ + "shape":"ObjectKey", + "documentation":"

The object key of the newly created object.

" + }, + "Expiration": { + "shape": "Expiration", + "documentation": "

If the expiration is configured for the object (see PutBucketLifecycleConfiguration), the response includes this header. It includes the expiry-date and rule-id key-value pairs that provide information about object expiration. The value of the rule-id is URL-encoded.

", + "location": "header", + "locationName": "x-amz-expiration" + }, + "ETag":{ + "shape":"ETag", + "documentation":"

Entity tag that identifies the newly created object's data. Objects with different object data will have different entity tags. The entity tag is an opaque string. The entity tag may or may not be an MD5 digest of the object data. If the entity tag is not an MD5 digest of the object data, it will contain one or more nonhexadecimal characters and/or will consist of less than 32 or more than 32 hexadecimal digits. For more information about how the entity tag is calculated, see Checking object integrity in the Amazon S3 User Guide.

" + }, + "ETagHeader":{ + "shape":"ETag", + "documentation":"

Entity tag that identifies the newly created object's data. Objects with different object data will have different entity tags. The entity tag is an opaque string. The entity tag may or may not be an MD5 digest of the object data. If the entity tag is not an MD5 digest of the object data, it will contain one or more nonhexadecimal characters and/or will consist of less than 32 or more than 32 hexadecimal digits. For more information about how the entity tag is calculated, see Checking object integrity in the Amazon S3 User Guide.

", + "location": "header", + "locationName": "ETag" + }, + "ChecksumCRC32": { + "shape": "ChecksumCRC32", + "documentation": "

The base64-encoded, 32-bit CRC32 checksum of the object. This will only be present if it was uploaded with the object. With multipart uploads, this may not be a checksum value of the object. For more information about how checksums are calculated with multipart uploads, see Checking object integrity in the Amazon S3 User Guide.

", + "location": "header", + "locationName": "x-amz-checksum-crc32" + }, + "ChecksumCRC32C": { + "shape": "ChecksumCRC32C", + "documentation": "

The base64-encoded, 32-bit CRC32C checksum of the object. This will only be present if it was uploaded with the object. With multipart uploads, this may not be a checksum value of the object. For more information about how checksums are calculated with multipart uploads, see Checking object integrity in the Amazon S3 User Guide.

", + "location": "header", + "locationName": "x-amz-checksum-crc32c" + }, + "ChecksumSHA1": { + "shape": "ChecksumSHA1", + "documentation": "

The base64-encoded, 160-bit SHA-1 digest of the object. This will only be present if it was uploaded with the object. With multipart uploads, this may not be a checksum value of the object. For more information about how checksums are calculated with multipart uploads, see Checking object integrity in the Amazon S3 User Guide.

", + "location": "header", + "locationName": "x-amz-checksum-sha1" + }, + "ChecksumSHA256": { + "shape": "ChecksumSHA256", + "documentation": "

The base64-encoded, 256-bit SHA-256 digest of the object. This will only be present if it was uploaded with the object. With multipart uploads, this may not be a checksum value of the object. For more information about how checksums are calculated with multipart uploads, see Checking object integrity in the Amazon S3 User Guide.

", + "location": "header", + "locationName": "x-amz-checksum-sha256" + }, + "ServerSideEncryption": { + "shape": "ServerSideEncryption", + "documentation": "

If you specified server-side encryption either with an Amazon Web Services KMS key or Amazon S3-managed encryption key in your PUT request, the response includes this header. It confirms the encryption algorithm that Amazon S3 used to encrypt the object.

", + "location": "header", + "locationName": "x-amz-server-side-encryption" + }, + "VersionId": { + "shape": "ObjectVersionId", + "documentation": "

Version of the object.

", + "location": "header", + "locationName": "x-amz-version-id" + }, + "SSECustomerAlgorithm": { + "shape": "SSECustomerAlgorithm", + "documentation": "

If server-side encryption with a customer-provided encryption key was requested, the response will include this header confirming the encryption algorithm used.

", + "location": "header", + "locationName": "x-amz-server-side-encryption-customer-algorithm" + }, + "SSECustomerKeyMD5": { + "shape": "SSECustomerKeyMD5", + "documentation": "

If server-side encryption with a customer-provided encryption key was requested, the response will include this header to provide round-trip message integrity verification of the customer-provided encryption key.

", + "location": "header", + "locationName": "x-amz-server-side-encryption-customer-key-MD5" + }, + "SSEKMSKeyId": { + "shape": "SSEKMSKeyId", + "documentation": "

If x-amz-server-side-encryption is present and has the value of aws:kms, this header specifies the ID of the Amazon Web Services Key Management Service (Amazon Web Services KMS) symmetric customer managed key that was used for the object.

", + "location": "header", + "locationName": "x-amz-server-side-encryption-aws-kms-key-id" + }, + "SSEKMSEncryptionContext": { + "shape": "SSEKMSEncryptionContext", + "documentation": "

If present, specifies the Amazon Web Services KMS Encryption Context to use for object encryption. The value of this header is a base64-encoded UTF-8 string holding JSON with the encryption context key-value pairs.

", + "location": "header", + "locationName": "x-amz-server-side-encryption-context" + }, + "BucketKeyEnabled": { + "shape": "BucketKeyEnabled", + "documentation": "

Indicates whether the uploaded object uses an S3 Bucket Key for server-side encryption with Amazon Web Services KMS (SSE-KMS).

", + "location": "header", + "locationName": "x-amz-server-side-encryption-bucket-key-enabled" + }, + "RequestCharged": { + "shape": "RequestCharged", + "location": "header", + "locationName": "x-amz-request-charged" + } + } + } + }, { "op": "add", "path": "/shapes/NoSuchWebsiteConfiguration", diff --git a/localstack/services/s3/presigned_url.py b/localstack/services/s3/presigned_url.py index 3a3d8077cf8f8..3a45356ff8153 100644 --- a/localstack/services/s3/presigned_url.py +++ b/localstack/services/s3/presigned_url.py @@ -1,9 +1,11 @@ +import base64 import copy import datetime +import json import logging import re import time -from typing import Dict, Tuple, TypedDict, Union +from typing import Dict, List, Tuple, TypedDict, Union from urllib import parse as urlparse from botocore.auth import HmacV1QueryAuth, S3SigV4QueryAuth @@ -12,19 +14,25 @@ from botocore.credentials import Credentials, ReadOnlyCredentials from botocore.exceptions import NoCredentialsError from botocore.utils import percent_encode_sequence -from werkzeug.datastructures import Headers +from werkzeug.datastructures import Headers, ImmutableMultiDict from localstack import config from localstack.aws.api import RequestContext from localstack.aws.api.s3 import ( AccessDenied, AuthorizationQueryParametersError, + InvalidArgument, SignatureDoesNotMatch, ) from localstack.aws.chain import HandlerChain from localstack.constants import TEST_AWS_ACCESS_KEY_ID, TEST_AWS_SECRET_ACCESS_KEY from localstack.http import Request, Response -from localstack.services.s3.utils import S3_VIRTUAL_HOST_FORWARDED_HEADER, uses_host_addressing +from localstack.services.s3.utils import ( + S3_VIRTUAL_HOST_FORWARDED_HEADER, + _create_invalid_argument_exc, + capitalize_header_name_from_snake_case, + uses_host_addressing, +) from localstack.utils.strings import to_bytes LOG = logging.getLogger(__name__) @@ -41,6 +49,19 @@ "X-Amz-Signature", ] + +SIGNATURE_V2_POST_FIELDS = [ + "signature", + "AWSAccessKeyId", +] + +SIGNATURE_V4_POST_FIELDS = [ + "x-amz-signature", + "x-amz-algorithm", + "x-amz-credential", + "x-amz-date", +] + # headers to blacklist from request_dict.signed_headers BLACKLISTED_HEADERS = ["X-Amz-Security-Token"] @@ -66,6 +87,10 @@ HOST_COMBINATION_REGEX = r"^(.*)(:[\d]{0,6})" PORT_REPLACEMENT = [":80", ":443", ":%s" % config.EDGE_PORT, ""] +# STS policy expiration date format +POLICY_EXPIRATION_FORMAT1 = "%Y-%m-%dT%H:%M:%SZ" +POLICY_EXPIRATION_FORMAT2 = "%Y-%m-%dT%H:%M:%S.%fZ" + class NotValidSigV4Signature(TypedDict): signature_provided: str @@ -404,9 +429,9 @@ def _prepare_request_for_sig_v4_signature( if header_low in IGNORED_SIGV4_HEADERS: continue if header_low not in signed_headers.lower(): - not_signed_headers.append(header) + not_signed_headers.append(header_low) if header_low in signed_headers: - signature_headers[header] = value + signature_headers[header_low] = value if not_signed_headers: ex: AccessDenied = create_access_denied_headers_not_signed(", ".join(not_signed_headers)) @@ -595,3 +620,101 @@ def _find_valid_signature_through_ports(context: RequestContext) -> FindSigV4Res # Return the last values returned by the loop, not sure which one we should select return None, exception + + +def validate_post_policy(request_form: ImmutableMultiDict) -> None: + """ + Validate the pre-signed POST with its policy contained + For now, only validates its expiration + SigV2: https://docs.aws.amazon.com/AmazonS3/latest/userguide/HTTPPOSTExamples.html + SigV4: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-authentication-HTTPPOST.html + + :param request_form: the form data contained in the pre-signed POST request + :raises AccessDenied, SignatureDoesNotMatch + :return: None + """ + if not request_form.get("key"): + ex: InvalidArgument = _create_invalid_argument_exc( + message="Bucket POST must contain a field named 'key'. If it is specified, please check the order of the fields.", + name="key", + value="", + host_id=FAKE_HOST_ID, + ) + raise ex + + if not (policy := request_form.get("policy")): + # A POST request needs a policy except if the bucket is publicly writable + return + + # TODO: this does validation of fields only for now + is_v4 = _is_match_with_signature_fields(request_form, SIGNATURE_V4_POST_FIELDS) + is_v2 = _is_match_with_signature_fields(request_form, SIGNATURE_V2_POST_FIELDS) + if not is_v2 and not is_v4: + ex: AccessDenied = AccessDenied("Access Denied") + ex.HostId = FAKE_HOST_ID + raise ex + + try: + policy_decoded = json.loads(base64.b64decode(policy).decode("utf-8")) + except ValueError: + # this means the policy has been tampered with + signature = request_form.get("signature") if is_v2 else request_form.get("x-amz-signature") + ex: SignatureDoesNotMatch = create_signature_does_not_match_sig_v2( + request_signature=signature, + string_to_sign=policy, + ) + raise ex + + if expiration := policy_decoded.get("expiration"): + if is_expired(_parse_policy_expiration_date(expiration)): + ex: AccessDenied = AccessDenied("Invalid according to Policy: Policy expired.") + ex.HostId = FAKE_HOST_ID + raise ex + + # TODO: validate the signature + # TODO: validate the request according to the policy + + +def _parse_policy_expiration_date(expiration_string: str) -> datetime.datetime: + """ + Parses the Policy Expiration datetime string + :param expiration_string: a policy expiration string, can be of 2 format: `2007-12-01T12:00:00.000Z` or + `2007-12-01T12:00:00Z` + :return: a datetime object representing the expiration datetime + """ + try: + dt = datetime.datetime.strptime(expiration_string, POLICY_EXPIRATION_FORMAT1) + except Exception: + dt = datetime.datetime.strptime(expiration_string, POLICY_EXPIRATION_FORMAT2) + + # both date formats assume a UTC timezone ('Z' suffix), but it's not parsed as tzinfo into the datetime object + dt = dt.replace(tzinfo=datetime.timezone.utc) + return dt + + +def _is_match_with_signature_fields( + request_form: ImmutableMultiDict, signature_fields: List[str] +) -> bool: + """ + Checks if the form contains at least one of the required fields passed in `signature_fields` + If it contains at least one field, validates it contains all of them or raises InvalidArgument + :param request_form: ImmutableMultiDict: the pre-signed POST request form + :param signature_fields: the field we want to validate against + :raises InvalidArgument + :return: False if none of the fields are present, or True if it does + """ + if any(p in request_form for p in signature_fields): + for p in signature_fields: + if p not in request_form: + LOG.info("POST pre-sign missing fields") + argument_name = capitalize_header_name_from_snake_case(p) if "-" in p else p + ex: InvalidArgument = _create_invalid_argument_exc( + message=f"Bucket POST must contain a field named '{argument_name}'. If it is specified, please check the order of the fields.", + name=argument_name, + value="", + host_id=FAKE_HOST_ID, + ) + raise ex + + return True + return False diff --git a/localstack/services/s3/provider.py b/localstack/services/s3/provider.py index 49b4a2a08501a..4ad7b46c0401f 100644 --- a/localstack/services/s3/provider.py +++ b/localstack/services/s3/provider.py @@ -1,15 +1,26 @@ import copy import logging import os -from urllib.parse import SplitResult, quote, urlsplit, urlunsplit +from typing import IO +from urllib.parse import ( + SplitResult, + parse_qs, + quote, + urlencode, + urlparse, + urlsplit, + urlunparse, + urlunsplit, +) import moto.s3.responses as moto_s3_responses from localstack.aws.accounts import get_aws_account_id -from localstack.aws.api import CommonServiceException, RequestContext, handler +from localstack.aws.api import CommonServiceException, RequestContext, ServiceException, handler from localstack.aws.api.s3 import ( AccessControlPolicy, AccountId, + Body, BucketName, ChecksumAlgorithm, CompleteMultipartUploadOutput, @@ -23,6 +34,7 @@ DeleteObjectRequest, DeleteObjectTaggingOutput, DeleteObjectTaggingRequest, + ETag, GetBucketAclOutput, GetBucketLifecycleConfigurationOutput, GetBucketLifecycleOutput, @@ -44,6 +56,7 @@ NoSuchWebsiteConfiguration, NotificationConfiguration, ObjectKey, + PostResponse, PutBucketAclRequest, PutBucketLifecycleConfigurationRequest, PutBucketLifecycleRequest, @@ -71,6 +84,7 @@ from localstack.services.s3.presigned_url import ( s3_presigned_url_request_handler, s3_presigned_url_response_handler, + validate_post_policy, ) from localstack.services.s3.utils import ( ALLOWED_HEADER_OVERRIDES, @@ -78,6 +92,7 @@ VALID_ACL_PREDEFINED_GROUPS, VALID_GRANTEE_PERMISSIONS, _create_invalid_argument_exc, + capitalize_header_name_from_snake_case, get_bucket_from_moto, get_header_name, get_key_from_moto_bucket, @@ -572,6 +587,68 @@ def delete_bucket_website( # does not raise error if the bucket did not have a config, will simply return self.get_store().bucket_website_configuration.pop(bucket, None) + def post_object( + self, context: RequestContext, bucket: BucketName, body: IO[Body] = None + ) -> PostResponse: + # see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html + # TODO: signature validation is not implemented for pre-signed POST + # policy validation is not implemented either, except expiration and mandatory fields + validate_post_policy(context.request.form) + + # Botocore has trouble parsing responses with status code in the 3XX range, it interprets them as exception + # it then raises a nonsense one with a wrong code + # We have to create and populate the response manually if that happens + try: + response: PostResponse = call_moto(context=context) + except ServiceException as e: + if e.status_code == 303: + # the parser did not succeed in parsing the moto respond, we start constructing the response ourselves + response = PostResponse(StatusCode=e.status_code) + else: + raise e + + key_name = context.request.form.get("key") + if "${filename}" in key_name: + key_name = key_name.replace("${filename}", context.request.files["file"].filename) + + moto_backend = get_moto_s3_backend(context) + key = get_key_from_moto_bucket( + get_bucket_from_moto(moto_backend, bucket=bucket), key=key_name + ) + # hacky way to set the etag in the headers as well: two locations for one value + response["ETagHeader"] = key.etag + + if response["StatusCode"] == 303: + # we need to create the redirect, as the parser could not return the moto-calculated one + try: + redirect = _create_redirect_for_post_request( + base_redirect=context.request.form["success_action_redirect"], + bucket=bucket, + key=key_name, + etag=key.etag, + ) + response["LocationHeader"] = redirect + except ValueError: + # If S3 cannot interpret the URL, it acts as if the field is not present. + response["StatusCode"] = 204 + + response["LocationHeader"] = response.get( + "LocationHeader", f"{get_full_default_bucket_location(bucket)}{key_name}" + ) + + if bucket in self.get_store().bucket_versioning_status: + response["VersionId"] = key.version_id + + if context.request.form.get("success_action_status") != "201": + return response + + response["ETag"] = key.etag + response["Bucket"] = bucket + response["Key"] = key_name + response["Location"] = response["LocationHeader"] + + return response + def add_custom_routes(self): # virtual-host style: https://bucket-name.s3.region-code.amazonaws.com/key-name # host_pattern_vhost_style = f"{bucket}.s3.{LOCALHOST_HOSTNAME}:{get_edge_port_http()}" @@ -820,6 +897,37 @@ def is_object_expired(context: RequestContext, bucket: BucketName, key: ObjectKe return is_key_expired(key_object=key_object) +def _create_redirect_for_post_request( + base_redirect: str, bucket: BucketName, key: ObjectKey, etag: ETag +): + """ + POST requests can redirect if successful. It will take the URL provided and append query string parameters + (key, bucket and ETag). It needs to be a full URL. + :param base_redirect: the URL provided for redirection + :param bucket: bucket name + :param key: key name + :param etag: key ETag + :return: the URL provided with the new appended query string parameters + """ + parts = urlparse(base_redirect) + if not parts.netloc: + raise ValueError("The provided URL is not valid") + queryargs = parse_qs(parts.query) + queryargs["key"] = [key] + queryargs["bucket"] = [bucket] + queryargs["etag"] = [etag] + redirect_queryargs = urlencode(queryargs, doseq=True) + newparts = ( + parts.scheme, + parts.netloc, + parts.path, + parts.params, + redirect_queryargs, + parts.fragment, + ) + return urlunparse(newparts) + + def apply_moto_patches(): # importing here in case we need InvalidObjectState from `localstack.aws.api.s3` from moto.s3.exceptions import InvalidObjectState @@ -827,9 +935,6 @@ def apply_moto_patches(): if not os.environ.get("MOTO_S3_DEFAULT_KEY_BUFFER_SIZE"): os.environ["MOTO_S3_DEFAULT_KEY_BUFFER_SIZE"] = str(S3_MAX_FILE_SIZE_BYTES) - def _capitalize_header_name_from_snake_case(header_name: str) -> str: - return "-".join([part.capitalize() for part in header_name.split("-")]) - @patch(moto_s3_responses.S3Response.key_response) def _fix_key_response(fn, self, *args, **kwargs): """Change casing of Last-Modified headers to be picked by the parser""" @@ -842,7 +947,7 @@ def _fix_key_response(fn, self, *args, **kwargs): "content-encoding", ]: if header_value := resp_headers.pop(low_case_header, None): - header_name = _capitalize_header_name_from_snake_case(low_case_header) + header_name = capitalize_header_name_from_snake_case(low_case_header) resp_headers[header_name] = header_value return status_code, resp_headers, key_value diff --git a/localstack/services/s3/utils.py b/localstack/services/s3/utils.py index 941dfcf977345..69cfe74694928 100644 --- a/localstack/services/s3/utils.py +++ b/localstack/services/s3/utils.py @@ -180,9 +180,15 @@ def get_key_from_moto_bucket( def _create_invalid_argument_exc( - message: Union[str, None], name: str, value: str + message: Union[str, None], name: str, value: str, host_id: str = None ) -> InvalidArgument: ex = InvalidArgument(message) ex.ArgumentName = name ex.ArgumentValue = value + if host_id: + ex.HostId = host_id return ex + + +def capitalize_header_name_from_snake_case(header_name: str) -> str: + return "-".join([part.capitalize() for part in header_name.split("-")]) diff --git a/tests/integration/s3/test_s3.py b/tests/integration/s3/test_s3.py index f0241480235dc..c49caeba436d9 100644 --- a/tests/integration/s3/test_s3.py +++ b/tests/integration/s3/test_s3.py @@ -2212,7 +2212,8 @@ def test_put_object(self, s3_client, s3_bucket, snapshot): @pytest.mark.aws_validated @pytest.mark.xfail( - condition=not config.LEGACY_EDGE_PROXY, reason="failing with new HTTP gateway (only in CI)" + condition=not config.LEGACY_EDGE_PROXY and LEGACY_S3_PROVIDER, + reason="failing with new HTTP gateway (only in CI)", ) def test_post_object_with_files(self, s3_client, s3_bucket): object_key = "test-presigned-post-key" @@ -2220,7 +2221,10 @@ def test_post_object_with_files(self, s3_client, s3_bucket): body = b"something body" presigned_request = s3_client.generate_presigned_post( - Bucket=s3_bucket, Key=object_key, ExpiresIn=60 + Bucket=s3_bucket, + Key=object_key, + ExpiresIn=60, + Conditions=[{"bucket": s3_bucket}], ) # put object response = requests.post( @@ -2229,15 +2233,17 @@ def test_post_object_with_files(self, s3_client, s3_bucket): files={"file": body}, verify=False, ) - assert response.status_code == 204 + # get object and compare results downloaded_object = s3_client.get_object(Bucket=s3_bucket, Key=object_key) assert downloaded_object["Body"].read() == body @pytest.mark.aws_validated - def test_post_request_expires(self, s3_client, s3_bucket): - # TODO: failed against AWS + # old provider does not raise the right exception + @pytest.mark.skip_snapshot_verify(condition=is_old_provider) + def test_post_request_expires(self, s3_client, s3_bucket, snapshot): + snapshot.add_transformer(self._get_presigned_snapshot_transformers(snapshot)) # presign a post with a short expiry time object_key = "test-presigned-post-key" @@ -2256,12 +2262,145 @@ def test_post_request_expires(self, s3_client, s3_bucket): verify=False, ) - # should be AccessDenied instead of expired? - # expired Token must be about the identity token?? - - # FIXME: localstack returns 400 but aws returns 403 + exception = xmltodict.parse(response.content) + exception["StatusCode"] = response.status_code + snapshot.match("exception", exception) assert response.status_code in [400, 403] - assert "ExpiredToken" in response.text + + @pytest.mark.aws_validated + @pytest.mark.xfail( + condition=LEGACY_S3_PROVIDER, reason="Policy is not validated in legacy provider" + ) + @pytest.mark.parametrize( + "signature_version", + ["s3", "s3v4"], + ) + def test_post_request_malformed_policy(self, s3_client, s3_bucket, snapshot, signature_version): + snapshot.add_transformer(self._get_presigned_snapshot_transformers(snapshot)) + object_key = "test-presigned-malformed-policy" + + presigned_client = _s3_client_custom_config( + Config(signature_version=signature_version), + endpoint_url=_endpoint_url(), + ) + + presigned_request = presigned_client.generate_presigned_post( + Bucket=s3_bucket, Key=object_key, ExpiresIn=60 + ) + + # modify the base64 string to be wrong + original_policy = presigned_request["fields"]["policy"] + presigned_request["fields"]["policy"] = original_policy[:-2] + + response = requests.post( + presigned_request["url"], + data=presigned_request["fields"], + files={"file": "file content"}, + verify=False, + ) + # the policy has been modified, so the signature does not correspond + exception = xmltodict.parse(response.content) + exception["StatusCode"] = response.status_code + snapshot.match("exception-policy", exception) + # assert fields that snapshot cannot match + signature_field = "signature" if signature_version == "s3" else "x-amz-signature" + assert ( + exception["Error"]["SignatureProvided"] == presigned_request["fields"][signature_field] + ) + assert exception["Error"]["StringToSign"] == presigned_request["fields"]["policy"] + + @pytest.mark.aws_validated + @pytest.mark.xfail( + condition=LEGACY_S3_PROVIDER, reason="Signature is not validated in legacy provider" + ) + @pytest.mark.parametrize( + "signature_version", + ["s3", "s3v4"], + ) + def test_post_request_missing_signature( + self, s3_client, s3_bucket, snapshot, signature_version + ): + snapshot.add_transformer(self._get_presigned_snapshot_transformers(snapshot)) + object_key = "test-presigned-missing-signature" + + presigned_client = _s3_client_custom_config( + Config(signature_version=signature_version), + endpoint_url=_endpoint_url(), + ) + + presigned_request = presigned_client.generate_presigned_post( + Bucket=s3_bucket, Key=object_key, ExpiresIn=60 + ) + + # remove the signature field + signature_field = "signature" if signature_version == "s3" else "x-amz-signature" + presigned_request["fields"].pop(signature_field) + + response = requests.post( + presigned_request["url"], + data=presigned_request["fields"], + files={"file": "file content"}, + verify=False, + ) + + # AWS seems to detected what kind of signature is missing from the policy fields + exception = xmltodict.parse(response.content) + exception["StatusCode"] = response.status_code + snapshot.match("exception-missing-signature", exception) + + @pytest.mark.aws_validated + @pytest.mark.xfail( + condition=LEGACY_S3_PROVIDER, reason="Policy is not validated in legacy provider" + ) + @pytest.mark.parametrize( + "signature_version", + ["s3", "s3v4"], + ) + def test_post_request_missing_fields(self, s3_client, s3_bucket, snapshot, signature_version): + snapshot.add_transformer(self._get_presigned_snapshot_transformers(snapshot)) + object_key = "test-presigned-missing-fields" + + presigned_client = _s3_client_custom_config( + Config(signature_version=signature_version), + endpoint_url=_endpoint_url(), + ) + + presigned_request = presigned_client.generate_presigned_post( + Bucket=s3_bucket, Key=object_key, ExpiresIn=60 + ) + + # remove some signature related fields + if signature_version == "s3": + presigned_request["fields"].pop("AWSAccessKeyId") + else: + presigned_request["fields"].pop("x-amz-algorithm") + presigned_request["fields"].pop("x-amz-credential") + + response = requests.post( + presigned_request["url"], + data=presigned_request["fields"], + files={"file": "file content"}, + verify=False, + ) + + exception = xmltodict.parse(response.content) + exception["StatusCode"] = response.status_code + snapshot.match("exception-missing-fields", exception) + + # pop everything else to see what exception comes back + presigned_request["fields"] = { + k: v for k, v in presigned_request["fields"].items() if k in ("key", "policy") + } + response = requests.post( + presigned_request["url"], + data=presigned_request["fields"], + files={"file": "file content"}, + verify=False, + ) + + exception = xmltodict.parse(response.content) + exception["StatusCode"] = response.status_code + snapshot.match("exception-no-sig-related-fields", exception) @pytest.mark.aws_validated def test_delete_has_empty_content_length_header(self, s3_client, s3_bucket): @@ -2413,7 +2552,7 @@ def test_put_object_with_md5_and_chunk_signature_bad_headers( snapshot.match("with-decoded-content-length", exception) # old provider does not raise the right error message - if LEGACY_S3_PROVIDER or (signature_version == "s3" and is_aws_cloud()): + if LEGACY_S3_PROVIDER or signature_version == "s3": assert b"SignatureDoesNotMatch" in result.content # we are either using s3v4 with new provider or whichever signature against AWS else: @@ -2426,7 +2565,7 @@ def test_put_object_with_md5_and_chunk_signature_bad_headers( if snapshotted: exception = xmltodict.parse(result.content) snapshot.match("without-decoded-content-length", exception) - if LEGACY_S3_PROVIDER or (signature_version == "s3" and is_aws_cloud()): + if LEGACY_S3_PROVIDER or signature_version == "s3": assert b"SignatureDoesNotMatch" in result.content else: assert b"AccessDenied" in result.content @@ -2728,7 +2867,6 @@ def test_s3_get_response_content_type_same_as_upload_and_range(self, s3_client, def test_s3_presigned_post_success_action_status_201_response(self, s3_client, s3_bucket): # a security policy is required if the bucket is not publicly writable # see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html#RESTObjectPOST-requests-form-fields - # TODO need to create new operation in the specs to handle presigned POST body = "something body" # get presigned URL object_key = "key-${filename}" @@ -2746,28 +2884,87 @@ def test_s3_presigned_post_success_action_status_201_response(self, s3_client, s files=files, verify=False, ) - # test + assert response.status_code == 201 json_response = xmltodict.parse(response.content) assert "PostResponse" in json_response json_response = json_response["PostResponse"] - # fixme 201 response is hardcoded - # see localstack.services.s3.s3_listener.ProxyListenerS3.get_201_response - if is_aws_cloud(): - location = f"{_bucket_url_vhost(s3_bucket, aws_stack.get_region())}/key-my-file" - etag = '"43281e21fce675ac3bcb3524b38ca4ed"' # TODO check quoting of etag - else: - # TODO: this location is very wrong + + if LEGACY_S3_PROVIDER and not is_aws_cloud(): + # legacy provider is does not manage PostResponse adequately location = "http://localhost/key-my-file" etag = "d41d8cd98f00b204e9800998ecf8427f" + else: + location = f"{_bucket_url_vhost(s3_bucket, aws_stack.get_region())}/key-my-file" + etag = '"43281e21fce675ac3bcb3524b38ca4ed"' + assert response.headers["ETag"] == etag + assert response.headers["Location"] == location + assert json_response["Location"] == location assert json_response["Bucket"] == s3_bucket assert json_response["Key"] == "key-my-file" assert json_response["ETag"] == etag + @pytest.mark.aws_validated + @pytest.mark.xfail(condition=LEGACY_S3_PROVIDER, reason="not supported in legacy provider") + def test_s3_presigned_post_success_action_redirect(self, s3_client, s3_bucket): + # a security policy is required if the bucket is not publicly writable + # see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html#RESTObjectPOST-requests-form-fields + body = "something body" + # get presigned URL + object_key = "key-test" + redirect_location = "http://localhost.test/random" + presigned_request = s3_client.generate_presigned_post( + Bucket=s3_bucket, + Key=object_key, + Fields={"success_action_redirect": redirect_location}, + Conditions=[ + {"bucket": s3_bucket}, + ["eq", "$success_action_redirect", redirect_location], + ], + ExpiresIn=60, + ) + files = {"file": ("my-file", body)} + response = requests.post( + presigned_request["url"], + data=presigned_request["fields"], + files=files, + verify=False, + allow_redirects=False, + ) + + assert response.status_code == 303 + assert not response.text + location = urlparse(response.headers["Location"]) + location_qs = parse_qs(location.query) + assert location_qs["key"][0] == object_key + assert location_qs["bucket"][0] == s3_bucket + assert location_qs["etag"][0] == '"43281e21fce675ac3bcb3524b38ca4ed"' + + # If S3 cannot interpret the URL, it acts as if the field is not present. + wrong_redirect = "/wrong/redirect/relative" + presigned_request = s3_client.generate_presigned_post( + Bucket=s3_bucket, + Key=object_key, + Fields={"success_action_redirect": wrong_redirect}, + Conditions=[ + {"bucket": s3_bucket}, + ["eq", "$success_action_redirect", wrong_redirect], + ], + ExpiresIn=60, + ) + response = requests.post( + presigned_request["url"], + data=presigned_request["fields"], + files=files, + verify=False, + allow_redirects=False, + ) + assert response.status_code == 204 + @pytest.mark.aws_validated def test_presigned_url_with_session_token(self, s3_create_bucket_with_client, sts_client): - # TODO we might be skipping signature validation here...... + # TODO we might be skipping signature validation here... bucket_name = f"bucket-{short_uid()}" key_name = "key" response = sts_client.get_session_token() diff --git a/tests/integration/s3/test_s3.snapshot.json b/tests/integration/s3/test_s3.snapshot.json index f243f3a1674b6..1751dab49165b 100644 --- a/tests/integration/s3/test_s3.snapshot.json +++ b/tests/integration/s3/test_s3.snapshot.json @@ -3292,6 +3292,156 @@ } } }, + "tests/integration/s3/test_s3.py::TestS3PresignedUrl::test_post_request_expires": { + "recorded-date": "04-10-2022, 17:59:26", + "recorded-content": { + "exception": { + "Error": { + "Code": "AccessDenied", + "HostId": "host-id", + "Message": "Invalid according to Policy: Policy expired.", + "RequestId": "" + }, + "StatusCode": 403 + } + } + }, + "tests/integration/s3/test_s3.py::TestS3PresignedUrl::test_post_request_malformed_policy": { + "recorded-date": "04-10-2022, 20:42:19", + "recorded-content": { + "exception": { + "Error": { + "AWSAccessKeyId": "", + "Code": "SignatureDoesNotMatch", + "HostId": "host-id", + "Message": "The request signature we calculated does not match the signature you provided. Check your key and signing method.", + "RequestId": "", + "SignatureProvided": "", + "StringToSign": "", + "StringToSignBytes": "" + }, + "StatusCode": 403 + } + } + }, + "tests/integration/s3/test_s3.py::TestS3PresignedUrl::test_post_request_malformed_policy[s3]": { + "recorded-date": "05-10-2022, 18:11:44", + "recorded-content": { + "exception-policy": { + "Error": { + "AWSAccessKeyId": "", + "Code": "SignatureDoesNotMatch", + "HostId": "host-id", + "Message": "The request signature we calculated does not match the signature you provided. Check your key and signing method.", + "RequestId": "", + "SignatureProvided": "", + "StringToSign": "", + "StringToSignBytes": "" + }, + "StatusCode": 403 + } + } + }, + "tests/integration/s3/test_s3.py::TestS3PresignedUrl::test_post_request_malformed_policy[s3v4]": { + "recorded-date": "05-10-2022, 18:11:46", + "recorded-content": { + "exception-policy": { + "Error": { + "AWSAccessKeyId": "", + "Code": "SignatureDoesNotMatch", + "HostId": "host-id", + "Message": "The request signature we calculated does not match the signature you provided. Check your key and signing method.", + "RequestId": "", + "SignatureProvided": "", + "StringToSign": "", + "StringToSignBytes": "" + }, + "StatusCode": 403 + } + } + }, + "tests/integration/s3/test_s3.py::TestS3PresignedUrl::test_post_request_missing_signature[s3]": { + "recorded-date": "05-10-2022, 18:14:22", + "recorded-content": { + "exception-missing-signature": { + "Error": { + "ArgumentName": "Signature", + "ArgumentValue": null, + "Code": "InvalidArgument", + "HostId": "host-id", + "Message": "Bucket POST must contain a field named 'Signature'. If it is specified, please check the order of the fields.", + "RequestId": "" + }, + "StatusCode": 400 + } + } + }, + "tests/integration/s3/test_s3.py::TestS3PresignedUrl::test_post_request_missing_signature[s3v4]": { + "recorded-date": "05-10-2022, 18:14:24", + "recorded-content": { + "exception-missing-signature": { + "Error": { + "ArgumentName": "X-Amz-Signature", + "ArgumentValue": null, + "Code": "InvalidArgument", + "HostId": "host-id", + "Message": "Bucket POST must contain a field named 'X-Amz-Signature'. If it is specified, please check the order of the fields.", + "RequestId": "" + }, + "StatusCode": 400 + } + } + }, + "tests/integration/s3/test_s3.py::TestS3PresignedUrl::test_post_request_missing_fields[s3]": { + "recorded-date": "05-10-2022, 18:29:23", + "recorded-content": { + "exception-missing-fields": { + "Error": { + "ArgumentName": "AWSAccessKeyId", + "ArgumentValue": null, + "Code": "InvalidArgument", + "HostId": "host-id", + "Message": "Bucket POST must contain a field named 'AWSAccessKeyId'. If it is specified, please check the order of the fields.", + "RequestId": "" + }, + "StatusCode": 400 + }, + "exception-no-sig-related-fields": { + "Error": { + "Code": "AccessDenied", + "HostId": "host-id", + "Message": "Access Denied", + "RequestId": "" + }, + "StatusCode": 403 + } + } + }, + "tests/integration/s3/test_s3.py::TestS3PresignedUrl::test_post_request_missing_fields[s3v4]": { + "recorded-date": "05-10-2022, 18:29:26", + "recorded-content": { + "exception-missing-fields": { + "Error": { + "ArgumentName": "X-Amz-Algorithm", + "ArgumentValue": null, + "Code": "InvalidArgument", + "HostId": "host-id", + "Message": "Bucket POST must contain a field named 'X-Amz-Algorithm'. If it is specified, please check the order of the fields.", + "RequestId": "" + }, + "StatusCode": 400 + }, + "exception-no-sig-related-fields": { + "Error": { + "Code": "AccessDenied", + "HostId": "host-id", + "Message": "Access Denied", + "RequestId": "" + }, + "StatusCode": 403 + } + } + }, "tests/integration/s3/test_s3.py::TestS3StaticWebsiteHosting::test_validate_website_configuration": { "recorded-date": "28-09-2022, 22:44:53", "recorded-content": {