localstack · thrau · Oct 10, 2022 · Oct 6, 2022 · Oct 3, 2022 · Oct 6, 2022
diff --git a/localstack/aws/api/s3/__init__.py b/localstack/aws/api/s3/__init__.py
@@ -600,6 +600,7 @@ class NoSuchUpload(ServiceException):
     code: str = "NoSuchUpload"
     sender_fault: bool = False
     status_code: int = 400
+    UploadId: Optional[MultipartUploadId]
 
 
 class ObjectAlreadyInActiveTierError(ServiceException):

diff --git a/localstack/aws/spec-patches.json b/localstack/aws/spec-patches.json
@@ -615,6 +615,13 @@
         "documentation": "<p>The specified bucket does not have a website configuration</p>",
         "exception": true
       }
+    },
+    {
+      "op": "add",
+      "path": "/shapes/NoSuchUpload/members/UploadId",
+      "value": {
+        "shape": "MultipartUploadId"
+      }
     }
   ]
 }
diff --git a/localstack/services/s3/models.py b/localstack/services/s3/models.py
@@ -13,7 +13,7 @@
     WebsiteConfiguration,
 )
 from localstack.constants import DEFAULT_AWS_ACCOUNT_ID
-from localstack.services.stores import AccountRegionBundle, BaseStore, LocalAttribute
+from localstack.services.stores import AccountRegionBundle, BaseStore, CrossRegionAttribute
 
 
 def get_moto_s3_backend(context: RequestContext = None) -> MotoS3Backend:
@@ -23,25 +23,27 @@ def get_moto_s3_backend(context: RequestContext = None) -> MotoS3Backend:
 
 class S3Store(BaseStore):
     # maps bucket name to bucket's list of notification configurations
-    bucket_notification_configs: Dict[BucketName, NotificationConfiguration] = LocalAttribute(
+    bucket_notification_configs: Dict[BucketName, NotificationConfiguration] = CrossRegionAttribute(
         default=dict
     )
 
     # maps bucket name to bucket's CORS settings
-    bucket_cors: Dict[BucketName, CORSConfiguration] = LocalAttribute(default=dict)
+    bucket_cors: Dict[BucketName, CORSConfiguration] = CrossRegionAttribute(default=dict)
 
     # maps bucket name to bucket's replication settings
-    bucket_replication: Dict[BucketName, ReplicationConfiguration] = LocalAttribute(default=dict)
+    bucket_replication: Dict[BucketName, ReplicationConfiguration] = CrossRegionAttribute(
+        default=dict
+    )
 
     # maps bucket name to bucket's lifecycle configuration
     # TODO: need to check "globality" of parameters / redirect
-    bucket_lifecycle_configuration: Dict[BucketName, BucketLifecycleConfiguration] = LocalAttribute(
-        default=dict
-    )
+    bucket_lifecycle_configuration: Dict[
+        BucketName, BucketLifecycleConfiguration
+    ] = CrossRegionAttribute(default=dict)
 
-    bucket_versioning_status: Dict[BucketName, bool] = LocalAttribute(default=dict)
+    bucket_versioning_status: Dict[BucketName, bool] = CrossRegionAttribute(default=dict)
 
-    bucket_website_configuration: Dict[BucketName, WebsiteConfiguration] = LocalAttribute(
+    bucket_website_configuration: Dict[BucketName, WebsiteConfiguration] = CrossRegionAttribute(
         default=dict
     )
 

diff --git a/localstack/services/s3/notifications.py b/localstack/services/s3/notifications.py
@@ -45,6 +45,7 @@
     "PutObject": Event.s3_ObjectCreated_Put,
     "CopyObject": Event.s3_ObjectCreated_Copy,
     "CompleteMultipartUpload": Event.s3_ObjectCreated_CompleteMultipartUpload,
+    "PostObject": Event.s3_ObjectCreated_Post,
     "PutObjectTagging": Event.s3_ObjectTagging_Put,
     "DeleteObjectTagging": Event.s3_ObjectTagging_Delete,
     "DeleteObject": Event.s3_ObjectRemoved_Delete,
@@ -89,12 +90,22 @@ class S3EventNotificationContext:
     key_version_id: str
 
     @classmethod
-    def from_request_context(cls, request_context: RequestContext) -> "S3EventNotificationContext":
+    def from_request_context(
+        cls, request_context: RequestContext, key_name: str = None
+    ) -> "S3EventNotificationContext":
+        """
+        Create an S3EventNotificationContext from a RequestContext.
+        The key is not always present in the request context depending on the event type. In that case, we can use
+        a provided one.
+        :param request_context: RequestContext
+        :param key_name: Optional, in case it's not provided in the RequestContext
+        :return: S3EventNotificationContext
+        """
         bucket_name = request_context.service_request["Bucket"]
         moto_backend = get_moto_s3_backend(request_context)
         bucket: FakeBucket = get_bucket_from_moto(moto_backend, bucket=bucket_name)
         key: FakeKey = get_key_from_moto_bucket(
-            moto_bucket=bucket, key=request_context.service_request["Key"]
+            moto_bucket=bucket, key=key_name or request_context.service_request["Key"]
         )
         return cls(
             event_type=EVENT_OPERATION_MAP.get(request_context.operation.wire_name, ""),
@@ -287,6 +298,7 @@ def _verify_target_exists(self, arn: str, arn_data: ArnData) -> None:
                 QueueName=arn_data["resource"], QueueOwnerAWSAccountId=arn_data["account"]
             )
         except ClientError:
+            LOG.exception("Could not validate the notification destination %s", arn)
             raise _create_invalid_argument_exc(
                 "Unable to validate the following destination configurations",
                 name=arn,

diff --git a/localstack/services/s3/presigned_url.py b/localstack/services/s3/presigned_url.py
@@ -385,7 +385,7 @@ def _reverse_inject_signature_hmac_v1_query(context: RequestContext) -> Request:
     for header, value in context.request.headers.items():
         header_low = header.lower()
         if header_low.startswith("x-amz-") or header_low in ["content-type", "date", "content-md5"]:
-            new_headers[header] = value
+            new_headers[header_low] = value
 
     # rebuild the query string
     new_query_string = percent_encode_sequence(new_query_string_dict)
@@ -707,7 +707,12 @@ def _is_match_with_signature_fields(
         for p in signature_fields:
             if p not in request_form:
                 LOG.info("POST pre-sign missing fields")
-                argument_name = capitalize_header_name_from_snake_case(p) if "-" in p else p
+                # .capitalize() does not work here, because of AWSAccessKeyId casing
+                argument_name = (
+                    capitalize_header_name_from_snake_case(p)
+                    if "-" in p
+                    else f"{p[0].upper()}{p[1:]}"
+                )
                 ex: InvalidArgument = _create_invalid_argument_exc(
                     message=f"Bucket POST must contain a field named '{argument_name}'.  If it is specified, please check the order of the fields.",
                     name=argument_name,

diff --git a/localstack/services/s3/provider.py b/localstack/services/s3/provider.py
@@ -1,7 +1,7 @@
 import copy
 import logging
 import os
-from typing import IO
+from typing import IO, Dict
 from urllib.parse import (
     SplitResult,
     parse_qs,
@@ -15,6 +15,7 @@
 
 import moto.s3.responses as moto_s3_responses
 
+from localstack import config
 from localstack.aws.accounts import get_aws_account_id
 from localstack.aws.api import CommonServiceException, RequestContext, ServiceException, handler
 from localstack.aws.api.s3 import (
@@ -42,8 +43,13 @@
     GetBucketRequestPaymentOutput,
     GetBucketRequestPaymentRequest,
     GetBucketWebsiteOutput,
+    GetObjectAttributesOutput,
+    GetObjectAttributesParts,
+    GetObjectAttributesRequest,
     GetObjectOutput,
     GetObjectRequest,
+    GetObjectTaggingOutput,
+    GetObjectTaggingRequest,
     HeadObjectOutput,
     HeadObjectRequest,
     InvalidBucketName,
@@ -72,7 +78,6 @@
 from localstack.aws.api.s3 import Type as GranteeType
 from localstack.aws.api.s3 import WebsiteConfiguration
 from localstack.aws.handlers import modify_service_response, serve_custom_service_request_handlers
-from localstack.config import get_edge_port_http, get_protocol
 from localstack.constants import LOCALHOST_HOSTNAME
 from localstack.http import Request, Response
 from localstack.http.proxy import forward
@@ -136,7 +141,9 @@ def __init__(self, message=None):
 
 
 def get_full_default_bucket_location(bucket_name):
-    return f"{get_protocol()}://{bucket_name}.s3.{LOCALHOST_HOSTNAME}:{get_edge_port_http()}/"
+    if config.HOSTNAME_EXTERNAL != config.LOCALHOST:
+        return f"{config.get_protocol()}://{config.HOSTNAME_EXTERNAL}:{config.get_edge_port_http()}/{bucket_name}/"
+    return f"{config.get_protocol()}://{bucket_name}.s3.{LOCALHOST_HOSTNAME}:{config.get_edge_port_http()}/"
 
 
 class S3Provider(S3Api, ServiceLifecycleHook):
@@ -162,11 +169,18 @@ def __init__(self) -> None:
     def on_before_stop(self):
         self._notification_dispatcher.shutdown()
 
-    def _notify(self, context: RequestContext, s3_notif_ctx: S3EventNotificationContext = None):
+    def _notify(
+        self,
+        context: RequestContext,
+        s3_notif_ctx: S3EventNotificationContext = None,
+        key_name: ObjectKey = None,
+    ):
         # we can provide the s3_event_notification_context, so in case of deletion of keys, we can create it before
         # it happens
         if not s3_notif_ctx:
-            s3_notif_ctx = S3EventNotificationContext.from_request_context(context)
+            s3_notif_ctx = S3EventNotificationContext.from_request_context(
+                context, key_name=key_name
+            )
         if notification_config := self.get_store().bucket_notification_configs.get(
             s3_notif_ctx.bucket_name
         ):
@@ -357,9 +371,25 @@ def complete_multipart_upload(
         self, context: RequestContext, request: CompleteMultipartUploadRequest
     ) -> CompleteMultipartUploadOutput:
         response: CompleteMultipartUploadOutput = call_moto(context)
+        # moto return the Location in AWS `http://{bucket}.s3.amazonaws.com/{key}`
+        response[
+            "Location"
+        ] = f'{get_full_default_bucket_location(request["Bucket"])}{response["Key"]}'
         self._notify(context)
         return response
 
+    @handler("GetObjectTagging", expand=False)
+    def get_object_tagging(
+        self, context: RequestContext, request: GetObjectTaggingRequest
+    ) -> GetObjectTaggingOutput:
+        response: GetObjectTaggingOutput = call_moto(context)
+        if (
+            "VersionId" in response
+            and request["Bucket"] not in self.get_store().bucket_versioning_status
+        ):
+            response.pop("VersionId")
+        return response
+
     @handler("PutObjectTagging", expand=False)
     def put_object_tagging(
         self, context: RequestContext, request: PutObjectTaggingRequest
@@ -639,6 +669,7 @@ def post_object(
         if bucket in self.get_store().bucket_versioning_status:
             response["VersionId"] = key.version_id
 
+        self._notify(context, key_name=key_name)
         if context.request.form.get("success_action_status") != "201":
             return response
 
@@ -649,6 +680,38 @@ def post_object(
 
         return response
 
+    @handler("GetObjectAttributes", expand=False)
+    def get_object_attributes(
+        self,
+        context: RequestContext,
+        request: GetObjectAttributesRequest,
+    ) -> GetObjectAttributesOutput:
+        bucket_name = request["Bucket"]
+        moto_backend = get_moto_s3_backend(context)
+        bucket = get_bucket_from_moto(moto_backend, bucket_name)
+        key = get_key_from_moto_bucket(moto_bucket=bucket, key=request["Key"])
+
+        object_attrs = request.get("ObjectAttributes", [])
+        response = GetObjectAttributesOutput()
+        # TODO: see Checksum field
+        if "ETag" in object_attrs:
+            response["ETag"] = key.etag.strip('"')
+        if "StorageClass" in object_attrs:
+            response["StorageClass"] = key.storage_class
+        if "ObjectSize" in object_attrs:
+            response["ObjectSize"] = key.size
+
+        response["LastModified"] = key.last_modified
+        if version_id := request.get("VersionId"):
+            response["VersionId"] = version_id
+
+        if key.multipart:
+            response["ObjectParts"] = GetObjectAttributesParts(
+                TotalPartsCount=len(key.multipart.partlist)
+            )
+
+        return response
+
     def add_custom_routes(self):
         # virtual-host style: https://bucket-name.s3.region-code.amazonaws.com/key-name
         # host_pattern_vhost_style = f"{bucket}.s3.<regex('({AWS_REGION_REGEX}\.)?'):region>{LOCALHOST_HOSTNAME}:{get_edge_port_http()}"
@@ -991,6 +1054,18 @@ def _fix_owner_id_list_bucket(fn, *args, **kwargs) -> str:
         )
         return res
 
+    @patch(moto_s3_responses.S3Response._tagging_from_xml)
+    def _fix_tagging_from_xml(fn, *args, **kwargs) -> Dict[str, str]:
+        """
+        Moto tries to parse the TagSet and then iterate of it, not checking if it returned something
+        Potential to be an easy upstream fix
+        """
+        try:
+            tags: Dict[str, str] = fn(*args, **kwargs)
+        except TypeError:
+            tags = {}
+        return tags
+
 
 def register_custom_handlers():
     serve_custom_service_request_handlers.append(s3_presigned_url_request_handler)

diff --git a/localstack/services/s3/utils.py b/localstack/services/s3/utils.py
@@ -4,7 +4,7 @@
 
 import moto.s3.models as moto_s3_models
 from moto.s3.exceptions import MissingBucket
-from moto.s3.models import FakeKey
+from moto.s3.models import FakeDeleteMarker, FakeKey
 
 from localstack.aws.api import ServiceException
 from localstack.aws.api.s3 import (
@@ -111,8 +111,8 @@ def verify_checksum(checksum_algorithm: str, data: bytes, request: Dict):
         )
 
 
-def is_key_expired(key_object: FakeKey) -> bool:
-    if not key_object or not key_object._expiry:
+def is_key_expired(key_object: Union[FakeKey, FakeDeleteMarker]) -> bool:
+    if not key_object or isinstance(key_object, FakeDeleteMarker) or not key_object._expiry:
         return False
     return key_object._expiry <= datetime.datetime.now(key_object._expiry.tzinfo)
 

diff --git a/tests/integration/cloudformation/test_cloudformation_legacy.py b/tests/integration/cloudformation/test_cloudformation_legacy.py
@@ -365,7 +365,8 @@ def test_cfn_handle_s3_notification_configuration(
         s3_client = create_boto_client("s3", region_name=region)
         bucket_name = f"target-{short_uid()}"
         queue_name = f"queue-{short_uid()}"
-        queue_arn = aws_stack.sqs_queue_arn(queue_name, region_name=s3_client.meta.region_name)
+        # the queue is always created in us-east-1
+        queue_arn = aws_stack.sqs_queue_arn(queue_name)
         if create_bucket_first:
             s3_client.create_bucket(
                 Bucket=bucket_name,