From 4100e960e8e05f7ce916391a980d57fe6cd0df6a Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Tue, 25 Oct 2022 10:28:31 +0200 Subject: [PATCH 01/11] fix internal botocore API usage --- localstack/aws/forwarder.py | 7 ++++++- tests/unit/aws/test_service_router.py | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/localstack/aws/forwarder.py b/localstack/aws/forwarder.py index a8e34a3d34bd2..99d6d6c158acd 100644 --- a/localstack/aws/forwarder.py +++ b/localstack/aws/forwarder.py @@ -164,7 +164,12 @@ def create_aws_request_context( "has_streaming_input": operation.has_streaming_input, "auth_type": operation.auth_type, } - request_dict = client._convert_to_request_dict(parameters, operation, context=request_context) + # The endpoint URL is mandatory here, set a dummy if not given + if not endpoint_url: + endpoint_url = "http://example.com" + request_dict = client._convert_to_request_dict( + parameters, operation, endpoint_url, context=request_context + ) aws_request = client._endpoint.create_request(request_dict, operation) context = RequestContext() diff --git a/tests/unit/aws/test_service_router.py b/tests/unit/aws/test_service_router.py index d62ef817f6a2f..8d7455db6aeff 100644 --- a/tests/unit/aws/test_service_router.py +++ b/tests/unit/aws/test_service_router.py @@ -164,7 +164,9 @@ def test_service_router_works_for_every_service( "auth_type": operation.auth_type, } request_args = _create_dummy_request_args(operation) - request_dict = client._convert_to_request_dict(request_args, operation, request_context) + request_dict = client._convert_to_request_dict( + request_args, operation, "http://dummy-endpoint", request_context + ) request_object = create_request_object(request_dict) client._request_signer.sign(operation.name, request_object) request: Request = _botocore_request_to_localstack_request(request_object) From 60c521f3dbfc7ac9f5dd33f16aaf101b1bfe5067 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Tue, 25 Oct 2022 10:49:49 +0200 Subject: [PATCH 02/11] adjust dummy endpoint --- localstack/aws/forwarder.py | 2 +- tests/unit/aws/test_service_router.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/localstack/aws/forwarder.py b/localstack/aws/forwarder.py index 99d6d6c158acd..21c4cea0b95ac 100644 --- a/localstack/aws/forwarder.py +++ b/localstack/aws/forwarder.py @@ -166,7 +166,7 @@ def create_aws_request_context( } # The endpoint URL is mandatory here, set a dummy if not given if not endpoint_url: - endpoint_url = "http://example.com" + endpoint_url = "http://dummy-endpoint.com/" request_dict = client._convert_to_request_dict( parameters, operation, endpoint_url, context=request_context ) diff --git a/tests/unit/aws/test_service_router.py b/tests/unit/aws/test_service_router.py index 8d7455db6aeff..2d360343ef50c 100644 --- a/tests/unit/aws/test_service_router.py +++ b/tests/unit/aws/test_service_router.py @@ -165,7 +165,7 @@ def test_service_router_works_for_every_service( } request_args = _create_dummy_request_args(operation) request_dict = client._convert_to_request_dict( - request_args, operation, "http://dummy-endpoint", request_context + request_args, operation, "http://dummy-endpoint.com/", request_context ) request_object = create_request_object(request_dict) client._request_signer.sign(operation.name, request_object) From 549efc43182459d1c870c56a86970068d4713904 Mon Sep 17 00:00:00 2001 From: Thomas Rausch Date: Tue, 20 Dec 2022 10:56:05 +0100 Subject: [PATCH 03/11] upgrade botocore version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index dd573ad0b67d5..922d323f176ee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,7 +67,7 @@ runtime = awscli>=1.22.90 awscrt>=0.13.14 boto>=2.49.0 - botocore>=1.12.13,<1.28.0 + botocore>=1.28.0 cbor2>=5.2.0 crontab>=0.22.6 cryptography From 9759e22e4a6ae69bc3cc19995b4beeb2bb4b74f9 Mon Sep 17 00:00:00 2001 From: Stefanie Plieschnegger Date: Tue, 20 Dec 2022 12:14:36 +0100 Subject: [PATCH 04/11] upgrade boto3 version, to fix dependency issue with botocore --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 922d323f176ee..cc9554f4b8c00 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,7 +25,7 @@ packages=find: # dependencies that are required for the cli (via pip install localstack) install_requires = - boto3>=1.20,<1.25.0 + boto3>=1.25.0 click>=7.0 cachetools~=5.0.0 #dnspython==1.16.0 From 52c36f265dc3695755a81f107c82ebd78b98d4e4 Mon Sep 17 00:00:00 2001 From: Stefanie Plieschnegger Date: Thu, 22 Dec 2022 14:54:01 +0100 Subject: [PATCH 05/11] adapt list of not-supported services for service router tests --- tests/unit/aws/test_service_router.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/aws/test_service_router.py b/tests/unit/aws/test_service_router.py index 2d360343ef50c..da24000aa174a 100644 --- a/tests/unit/aws/test_service_router.py +++ b/tests/unit/aws/test_service_router.py @@ -30,6 +30,8 @@ def _collect_operations() -> Tuple[ServiceModel, OperationModel]: "chime-sdk-media-pipelines", "chime-sdk-meetings", "chime-sdk-messaging", + "chime-sdk-voice", + "codecatalyst", "connect", "connect-contact-lens", "greengrassv2", @@ -40,6 +42,7 @@ def _collect_operations() -> Tuple[ServiceModel, OperationModel]: "kinesis-video-archived-media", "kinesis-video-media", "kinesis-video-signaling", + "kinesis-video-webrtc-storage", "kinesisvideo", "lex-models", "lex-runtime", @@ -51,9 +54,11 @@ def _collect_operations() -> Tuple[ServiceModel, OperationModel]: "pinpoint-sms-voice", "sagemaker-edge", "sagemaker-featurestore-runtime", + "sagemaker-metrics", "sms-voice", "sso", "sso-oidc", + "workdocs", ]: yield pytest.param( service, From 2af024324a38a9720f97d2f619df40c21752a707 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Thu, 22 Dec 2022 20:08:45 +0100 Subject: [PATCH 06/11] debug: try to understand failure of s3 unit tests --- localstack/aws/protocol/op_router.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/localstack/aws/protocol/op_router.py b/localstack/aws/protocol/op_router.py index 1f914bc5be0f4..bcaf5ddd73ea7 100644 --- a/localstack/aws/protocol/op_router.py +++ b/localstack/aws/protocol/op_router.py @@ -1,3 +1,4 @@ +import logging import re from collections import defaultdict from typing import Any, AnyStr, Dict, List, Mapping, Match, NamedTuple, Optional, Tuple @@ -11,6 +12,8 @@ from localstack.http import Request from localstack.http.request import get_raw_path +LOG = logging.getLogger(__name__) + class GreedyPathConverter(PathConverter): regex = ".*?" @@ -226,11 +229,15 @@ def _create_service_map(service: ServiceModel) -> Map: rules = [] + is_s3 = service.service_name == "s3" + # group all operations by their path and method path_index: Dict[(str, str), List[_HttpOperation]] = defaultdict(list) for op in ops: http_op = _HttpOperation.from_operation(op) path_index[(http_op.path, http_op.method)].append(http_op) + if is_s3: + LOG.info("Operation %s -> HTTP op %s", op, http_op) # create a matching rule for each (path, method) combination for (path, method), ops in path_index.items(): @@ -243,6 +250,14 @@ def _create_service_map(service: ServiceModel) -> Map: op = ops[0] rules.append(_StrictMethodRule(string=rule_string, method=method, endpoint=op.operation)) # type: ignore else: + if is_s3: + LOG.info( + "ambiguity: path: '%s', rule_string '%s', method '%s', ops: '%s'", + path, + rule_string, + method, + ops, + ) # if there is an ambiguity with only the (path, method) combination, # a custom rule - which can use additional request metadata - needs to be used rules.append(_RequestMatchingRule(string=rule_string, method=method, operations=ops)) From 5366c752c4b157d10ff68a4db3fdc17970169687 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Thu, 22 Dec 2022 20:46:06 +0100 Subject: [PATCH 07/11] try dirty fix for unit tests --- localstack/aws/protocol/op_router.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/localstack/aws/protocol/op_router.py b/localstack/aws/protocol/op_router.py index bcaf5ddd73ea7..f6cfa6dc83194 100644 --- a/localstack/aws/protocol/op_router.py +++ b/localstack/aws/protocol/op_router.py @@ -31,7 +31,11 @@ class _HttpOperation(NamedTuple): @staticmethod def from_operation(op: OperationModel) -> "_HttpOperation": - uri = op.http.get("requestUri") + uri = ( + op.http.get("requestUri") + if op.service_model.service_name != "s3" + else op.http.get("authPath") + ) method = op.http.get("method") deprecated = op.deprecated From 833d74df9074c5a0985a6f0738ac0e9fcf8e1be4 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Thu, 22 Dec 2022 20:48:36 +0100 Subject: [PATCH 08/11] try dirty fix for unit tests - better --- localstack/aws/protocol/op_router.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/localstack/aws/protocol/op_router.py b/localstack/aws/protocol/op_router.py index f6cfa6dc83194..9c6ff0a2c3b82 100644 --- a/localstack/aws/protocol/op_router.py +++ b/localstack/aws/protocol/op_router.py @@ -31,11 +31,11 @@ class _HttpOperation(NamedTuple): @staticmethod def from_operation(op: OperationModel) -> "_HttpOperation": - uri = ( - op.http.get("requestUri") - if op.service_model.service_name != "s3" - else op.http.get("authPath") - ) + if "authPath" in op.http: + uri = op.http["authPath"].rstrip("/") + else: + uri = op.http.get("requestUri") + method = op.http.get("method") deprecated = op.deprecated From c0fe416daa27614886a79c99511343384a1eaf95 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Thu, 22 Dec 2022 21:24:36 +0100 Subject: [PATCH 09/11] dirty fix for serialize_to_request issue in test_parser.py --- tests/unit/aws/protocol/test_parser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/aws/protocol/test_parser.py b/tests/unit/aws/protocol/test_parser.py index 913dbe17ea257..0c4da8bad051b 100644 --- a/tests/unit/aws/protocol/test_parser.py +++ b/tests/unit/aws/protocol/test_parser.py @@ -302,6 +302,8 @@ def _botocore_parser_integration_test( operation_model = service.operation_model(action) serialized_request = serializer.serialize_to_request(kwargs, operation_model) + if "auth_path" in serialized_request: + serialized_request["url_path"] = serialized_request["auth_path"] prepare_request_dict(serialized_request, "") split_url = urlsplit(serialized_request.get("url")) path = split_url.path From e944982da9cbb4deb384b5e0233a69d31c5a626b Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Thu, 22 Dec 2022 23:17:32 +0100 Subject: [PATCH 10/11] fix integrations/test_moto.py --- localstack/aws/forwarder.py | 15 +++++++++-- localstack/aws/protocol/op_router.py | 22 ++++----------- tests/integration/test_moto.py | 37 +++++++++++++++----------- tests/unit/aws/protocol/test_parser.py | 8 ++++-- 4 files changed, 45 insertions(+), 37 deletions(-) diff --git a/localstack/aws/forwarder.py b/localstack/aws/forwarder.py index 21c4cea0b95ac..1231bb0278221 100644 --- a/localstack/aws/forwarder.py +++ b/localstack/aws/forwarder.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Mapping, Optional from urllib.parse import urlsplit -from botocore.awsrequest import AWSPreparedRequest +from botocore.awsrequest import AWSPreparedRequest, prepare_request_dict from botocore.config import Config as BotoConfig from werkzeug.datastructures import Headers @@ -170,8 +170,19 @@ def create_aws_request_context( request_dict = client._convert_to_request_dict( parameters, operation, endpoint_url, context=request_context ) - aws_request = client._endpoint.create_request(request_dict, operation) + if service_name == "s3" and (auth_path := request_dict.get("auth_path")): + request_dict["url_path"] = auth_path + # re-prepare the request dict (function called in _convert_to_request_dict) + # to use the correct url_path for s3 + prepare_request_dict( + request_dict, + endpoint_url=endpoint_url, + user_agent=client._client_config.user_agent, + context=request_context, + ) + + aws_request: AWSPreparedRequest = client._endpoint.create_request(request_dict, operation) context = RequestContext() context.service = service context.operation = operation diff --git a/localstack/aws/protocol/op_router.py b/localstack/aws/protocol/op_router.py index 9c6ff0a2c3b82..8ae364ba96d99 100644 --- a/localstack/aws/protocol/op_router.py +++ b/localstack/aws/protocol/op_router.py @@ -1,4 +1,3 @@ -import logging import re from collections import defaultdict from typing import Any, AnyStr, Dict, List, Mapping, Match, NamedTuple, Optional, Tuple @@ -12,8 +11,6 @@ from localstack.http import Request from localstack.http.request import get_raw_path -LOG = logging.getLogger(__name__) - class GreedyPathConverter(PathConverter): regex = ".*?" @@ -31,8 +28,11 @@ class _HttpOperation(NamedTuple): @staticmethod def from_operation(op: OperationModel) -> "_HttpOperation": - if "authPath" in op.http: - uri = op.http["authPath"].rstrip("/") + # following botocore update above 1.25, botocore will strip the bucket from the requestUri in the spec. + # it will then set the original requestUri to authPath. We need to use the original request if set, otherwise + # use the regular requestUri + if auth_path := op.http.get("authPath"): + uri = auth_path.rstrip("/") else: uri = op.http.get("requestUri") @@ -233,15 +233,11 @@ def _create_service_map(service: ServiceModel) -> Map: rules = [] - is_s3 = service.service_name == "s3" - # group all operations by their path and method path_index: Dict[(str, str), List[_HttpOperation]] = defaultdict(list) for op in ops: http_op = _HttpOperation.from_operation(op) path_index[(http_op.path, http_op.method)].append(http_op) - if is_s3: - LOG.info("Operation %s -> HTTP op %s", op, http_op) # create a matching rule for each (path, method) combination for (path, method), ops in path_index.items(): @@ -254,14 +250,6 @@ def _create_service_map(service: ServiceModel) -> Map: op = ops[0] rules.append(_StrictMethodRule(string=rule_string, method=method, endpoint=op.operation)) # type: ignore else: - if is_s3: - LOG.info( - "ambiguity: path: '%s', rule_string '%s', method '%s', ops: '%s'", - path, - rule_string, - method, - ops, - ) # if there is an ambiguity with only the (path, method) combination, # a custom rule - which can use additional request metadata - needs to be used rules.append(_RequestMatchingRule(string=rule_string, method=method, operations=ops)) diff --git a/tests/integration/test_moto.py b/tests/integration/test_moto.py index 389b55cc6c3d9..2e488a0c23975 100644 --- a/tests/integration/test_moto.py +++ b/tests/integration/test_moto.py @@ -81,34 +81,34 @@ def test_call_s3_with_streaming_trait(payload, monkeypatch): # In the absence of below patch, Moto and LocalStack uses difference AWS Account IDs causing the test to fail monkeypatch.setattr(localstack.aws.accounts, "DEFAULT_AWS_ACCOUNT_ID", DEFAULT_MOTO_ACCOUNT_ID) + def _call_moto_s3(action, params): + # we need to set the endpoint because moto will match the host to know if it's a path addressed or + # virtual host addressed bucket + # see moto.s3.responses.S3Response.subdomain_based_buckets + context = moto.create_aws_request_context( + "s3", action, params, endpoint_url="http://localstack.com" + ) + return moto.call_moto(context) + bucket_name = f"bucket-{short_uid()}" key_name = f"key-{short_uid()}" # create the bucket - moto.call_moto(moto.create_aws_request_context("s3", "CreateBucket", {"Bucket": bucket_name})) + _call_moto_s3("CreateBucket", {"Bucket": bucket_name}) - moto.call_moto( - moto.create_aws_request_context( - "s3", "PutObject", {"Bucket": bucket_name, "Key": key_name, "Body": payload} - ) - ) + _call_moto_s3("PutObject", {"Bucket": bucket_name, "Key": key_name, "Body": payload}) # check whether it was created/received correctly - response = moto.call_moto( - moto.create_aws_request_context("s3", "GetObject", {"Bucket": bucket_name, "Key": key_name}) - ) + response = _call_moto_s3("GetObject", {"Bucket": bucket_name, "Key": key_name}) + assert hasattr( response["Body"], "read" ), f"expected Body to be readable, was {type(response['Body'])}" assert response["Body"].read() == b"foobar" # cleanup - moto.call_moto( - moto.create_aws_request_context( - "s3", "DeleteObject", {"Bucket": bucket_name, "Key": key_name} - ) - ) - moto.call_moto(moto.create_aws_request_context("s3", "DeleteBucket", {"Bucket": bucket_name})) + _call_moto_s3("DeleteObject", {"Bucket": bucket_name, "Key": key_name}) + _call_moto_s3("DeleteBucket", {"Bucket": bucket_name}) def test_call_include_response_metadata(): @@ -299,7 +299,12 @@ def test_moto_fallback_dispatcher_error_handling(monkeypatch): dispatcher = MotoFallbackDispatcher(provider) def _dispatch(action, params): - context = moto.create_aws_request_context("s3", action, params) + # we need to set the endpoint because moto will match the host to know if it's a path addressed or + # virtual host addressed bucket + # see moto.s3.responses.S3Response.subdomain_based_buckets + context = moto.create_aws_request_context( + "s3", action, params, endpoint_url="http://localstack.com" + ) return dispatcher[action](context, params) bucket_name = f"bucket-{short_uid()}" diff --git a/tests/unit/aws/protocol/test_parser.py b/tests/unit/aws/protocol/test_parser.py index 0c4da8bad051b..eb6e3bfbe0ead 100644 --- a/tests/unit/aws/protocol/test_parser.py +++ b/tests/unit/aws/protocol/test_parser.py @@ -302,8 +302,12 @@ def _botocore_parser_integration_test( operation_model = service.operation_model(action) serialized_request = serializer.serialize_to_request(kwargs, operation_model) - if "auth_path" in serialized_request: - serialized_request["url_path"] = serialized_request["auth_path"] + # following botocore update above 1.25, botocore will strip the bucket from the requestUri in the spec. + # it will then set the original requestUri to authPath, that we can retrieve in the serialized_request + # we need to check if the original path is there, otherwise use the provided one + if auth_path := serialized_request.get("auth_path"): + serialized_request["url_path"] = auth_path + prepare_request_dict(serialized_request, "") split_url = urlsplit(serialized_request.get("url")) path = split_url.path From 33585508acd569b6ca592f9736f37676b91c0c7b Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Fri, 23 Dec 2022 13:18:48 +0100 Subject: [PATCH 11/11] cleanup code, more detailed comments --- localstack/aws/forwarder.py | 12 +++++---- localstack/aws/protocol/op_router.py | 6 ++--- tests/integration/test_moto.py | 37 +++++++++++--------------- tests/unit/aws/protocol/test_parser.py | 6 ++--- tests/unit/aws/test_service_router.py | 4 ++- 5 files changed, 32 insertions(+), 33 deletions(-) diff --git a/localstack/aws/forwarder.py b/localstack/aws/forwarder.py index 1231bb0278221..3dbe2f883d0a0 100644 --- a/localstack/aws/forwarder.py +++ b/localstack/aws/forwarder.py @@ -164,17 +164,19 @@ def create_aws_request_context( "has_streaming_input": operation.has_streaming_input, "auth_type": operation.auth_type, } - # The endpoint URL is mandatory here, set a dummy if not given + + # The endpoint URL is mandatory here, set a dummy if not given (doesn't _need_ to be localstack specific) if not endpoint_url: - endpoint_url = "http://dummy-endpoint.com/" + endpoint_url = "http://localhost.localstack.cloud" request_dict = client._convert_to_request_dict( parameters, operation, endpoint_url, context=request_context ) - if service_name == "s3" and (auth_path := request_dict.get("auth_path")): + if auth_path := request_dict.get("auth_path"): + # botocore >= 1.28 might modify the url path of the request dict (specifically for S3). + # It will then set the original url path as "auth_path". If the auth_path is set, we reset the url_path. + # Afterwards the request needs to be prepared again. request_dict["url_path"] = auth_path - # re-prepare the request dict (function called in _convert_to_request_dict) - # to use the correct url_path for s3 prepare_request_dict( request_dict, endpoint_url=endpoint_url, diff --git a/localstack/aws/protocol/op_router.py b/localstack/aws/protocol/op_router.py index 8ae364ba96d99..b9ea8a09f1646 100644 --- a/localstack/aws/protocol/op_router.py +++ b/localstack/aws/protocol/op_router.py @@ -28,9 +28,9 @@ class _HttpOperation(NamedTuple): @staticmethod def from_operation(op: OperationModel) -> "_HttpOperation": - # following botocore update above 1.25, botocore will strip the bucket from the requestUri in the spec. - # it will then set the original requestUri to authPath. We need to use the original request if set, otherwise - # use the regular requestUri + # botocore >= 1.28 might modify the internal model (specifically for S3). + # It will modify the request URI and set the original value at "authPath". + # Use authPath if set, otherwise use the regular requestUri. if auth_path := op.http.get("authPath"): uri = auth_path.rstrip("/") else: diff --git a/tests/integration/test_moto.py b/tests/integration/test_moto.py index 2e488a0c23975..389b55cc6c3d9 100644 --- a/tests/integration/test_moto.py +++ b/tests/integration/test_moto.py @@ -81,34 +81,34 @@ def test_call_s3_with_streaming_trait(payload, monkeypatch): # In the absence of below patch, Moto and LocalStack uses difference AWS Account IDs causing the test to fail monkeypatch.setattr(localstack.aws.accounts, "DEFAULT_AWS_ACCOUNT_ID", DEFAULT_MOTO_ACCOUNT_ID) - def _call_moto_s3(action, params): - # we need to set the endpoint because moto will match the host to know if it's a path addressed or - # virtual host addressed bucket - # see moto.s3.responses.S3Response.subdomain_based_buckets - context = moto.create_aws_request_context( - "s3", action, params, endpoint_url="http://localstack.com" - ) - return moto.call_moto(context) - bucket_name = f"bucket-{short_uid()}" key_name = f"key-{short_uid()}" # create the bucket - _call_moto_s3("CreateBucket", {"Bucket": bucket_name}) + moto.call_moto(moto.create_aws_request_context("s3", "CreateBucket", {"Bucket": bucket_name})) - _call_moto_s3("PutObject", {"Bucket": bucket_name, "Key": key_name, "Body": payload}) + moto.call_moto( + moto.create_aws_request_context( + "s3", "PutObject", {"Bucket": bucket_name, "Key": key_name, "Body": payload} + ) + ) # check whether it was created/received correctly - response = _call_moto_s3("GetObject", {"Bucket": bucket_name, "Key": key_name}) - + response = moto.call_moto( + moto.create_aws_request_context("s3", "GetObject", {"Bucket": bucket_name, "Key": key_name}) + ) assert hasattr( response["Body"], "read" ), f"expected Body to be readable, was {type(response['Body'])}" assert response["Body"].read() == b"foobar" # cleanup - _call_moto_s3("DeleteObject", {"Bucket": bucket_name, "Key": key_name}) - _call_moto_s3("DeleteBucket", {"Bucket": bucket_name}) + moto.call_moto( + moto.create_aws_request_context( + "s3", "DeleteObject", {"Bucket": bucket_name, "Key": key_name} + ) + ) + moto.call_moto(moto.create_aws_request_context("s3", "DeleteBucket", {"Bucket": bucket_name})) def test_call_include_response_metadata(): @@ -299,12 +299,7 @@ def test_moto_fallback_dispatcher_error_handling(monkeypatch): dispatcher = MotoFallbackDispatcher(provider) def _dispatch(action, params): - # we need to set the endpoint because moto will match the host to know if it's a path addressed or - # virtual host addressed bucket - # see moto.s3.responses.S3Response.subdomain_based_buckets - context = moto.create_aws_request_context( - "s3", action, params, endpoint_url="http://localstack.com" - ) + context = moto.create_aws_request_context("s3", action, params) return dispatcher[action](context, params) bucket_name = f"bucket-{short_uid()}" diff --git a/tests/unit/aws/protocol/test_parser.py b/tests/unit/aws/protocol/test_parser.py index eb6e3bfbe0ead..6fdd5d44689c8 100644 --- a/tests/unit/aws/protocol/test_parser.py +++ b/tests/unit/aws/protocol/test_parser.py @@ -302,9 +302,9 @@ def _botocore_parser_integration_test( operation_model = service.operation_model(action) serialized_request = serializer.serialize_to_request(kwargs, operation_model) - # following botocore update above 1.25, botocore will strip the bucket from the requestUri in the spec. - # it will then set the original requestUri to authPath, that we can retrieve in the serialized_request - # we need to check if the original path is there, otherwise use the provided one + + # botocore >= 1.28 might modify the url path of the request dict (specifically for S3). + # It will then set the original url path as "auth_path". If the auth_path is set, we reset the url_path. if auth_path := serialized_request.get("auth_path"): serialized_request["url_path"] = auth_path diff --git a/tests/unit/aws/test_service_router.py b/tests/unit/aws/test_service_router.py index da24000aa174a..38e062ab0f14f 100644 --- a/tests/unit/aws/test_service_router.py +++ b/tests/unit/aws/test_service_router.py @@ -169,8 +169,10 @@ def test_service_router_works_for_every_service( "auth_type": operation.auth_type, } request_args = _create_dummy_request_args(operation) + + # The endpoint URL is mandatory here, just set a dummy (doesn't _need_ to be localstack specific) request_dict = client._convert_to_request_dict( - request_args, operation, "http://dummy-endpoint.com/", request_context + request_args, operation, "http://localhost.localstack.cloud", request_context ) request_object = create_request_object(request_dict) client._request_signer.sign(operation.name, request_object)