From fbc0600be3b7906014c76f99392e49bfebb236a3 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Tue, 11 Jul 2023 11:10:11 +0200 Subject: [PATCH 1/6] fix s3 op router following botocore 1.31.2 --- localstack/aws/protocol/op_router.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/localstack/aws/protocol/op_router.py b/localstack/aws/protocol/op_router.py index d9be6aa269ab6..319baabee42bf 100644 --- a/localstack/aws/protocol/op_router.py +++ b/localstack/aws/protocol/op_router.py @@ -39,10 +39,13 @@ class _HttpOperation(NamedTuple): @staticmethod def from_operation(op: OperationModel) -> "_HttpOperation": # botocore >= 1.28 might modify the internal model (specifically for S3). - # It will modify the request URI and set the original value at "authPath". + # It will modify the request URI to strip the bucket name from the path and set the original value at + # "authPath". + # Since botocore 1.31.2, botocore will strip the query from the `authPart` + # We need to add it back from `requestUri` field # Use authPath if set, otherwise use the regular requestUri. if auth_path := op.http.get("authPath"): - uri = auth_path.rstrip("/") + uri = auth_path.rstrip("/") + op.http.get("requestUri", "").split("?")[-1] else: uri = op.http.get("requestUri") From 121b73f2f4c5d7abb73c3857df0f9914f48165ce Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Tue, 11 Jul 2023 11:29:44 +0200 Subject: [PATCH 2/6] upgrade botocore dep --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d79432c0a4b81..b2a3e7abd3944 100644 --- a/setup.cfg +++ b/setup.cfg @@ -70,7 +70,7 @@ runtime = awscrt>=0.13.14 boto>=2.49.0 boto3>=1.26.121 - botocore>=1.29.133 + botocore>=1.31.2 cbor2>=5.2.0 crontab>=0.22.6 dnslib>=0.9.10 From 456393f973ed0f9d14f2908aa7a067d75fef4146 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Tue, 11 Jul 2023 12:03:26 +0200 Subject: [PATCH 3/6] use partition instead of split --- localstack/aws/protocol/op_router.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/localstack/aws/protocol/op_router.py b/localstack/aws/protocol/op_router.py index 319baabee42bf..ba3eeb4c5d32b 100644 --- a/localstack/aws/protocol/op_router.py +++ b/localstack/aws/protocol/op_router.py @@ -45,7 +45,8 @@ def from_operation(op: OperationModel) -> "_HttpOperation": # We need to add it back from `requestUri` field # Use authPath if set, otherwise use the regular requestUri. if auth_path := op.http.get("authPath"): - uri = auth_path.rstrip("/") + op.http.get("requestUri", "").split("?")[-1] + query = op.http.get("requestUri", "").partition("?") + uri = auth_path.rstrip("/") + query[2] else: uri = op.http.get("requestUri") From ce393a8fd34744dfb1463fb743d16bf9df43be8b Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Tue, 11 Jul 2023 12:05:05 +0200 Subject: [PATCH 4/6] cleaner partition --- localstack/aws/protocol/op_router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/localstack/aws/protocol/op_router.py b/localstack/aws/protocol/op_router.py index ba3eeb4c5d32b..3d6f94e7a9117 100644 --- a/localstack/aws/protocol/op_router.py +++ b/localstack/aws/protocol/op_router.py @@ -45,8 +45,8 @@ def from_operation(op: OperationModel) -> "_HttpOperation": # We need to add it back from `requestUri` field # Use authPath if set, otherwise use the regular requestUri. if auth_path := op.http.get("authPath"): - query = op.http.get("requestUri", "").partition("?") - uri = auth_path.rstrip("/") + query[2] + path, sep, query = op.http.get("requestUri", "").partition("?") + uri = auth_path.rstrip("/") + query else: uri = op.http.get("requestUri") From d67f8efd576505598a73ffafa574d1fc57b57bb5 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Tue, 11 Jul 2023 12:41:03 +0200 Subject: [PATCH 5/6] re-add separator --- localstack/aws/protocol/op_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localstack/aws/protocol/op_router.py b/localstack/aws/protocol/op_router.py index 3d6f94e7a9117..3d4934f80c8b1 100644 --- a/localstack/aws/protocol/op_router.py +++ b/localstack/aws/protocol/op_router.py @@ -46,7 +46,7 @@ def from_operation(op: OperationModel) -> "_HttpOperation": # Use authPath if set, otherwise use the regular requestUri. if auth_path := op.http.get("authPath"): path, sep, query = op.http.get("requestUri", "").partition("?") - uri = auth_path.rstrip("/") + query + uri = f"{auth_path.rstrip('/')}{sep}{query}" else: uri = op.http.get("requestUri") From 7823ef3e6ba306584533abafe8705cf3e72a05ad Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Tue, 11 Jul 2023 14:10:18 +0200 Subject: [PATCH 6/6] fix fixture and create_aws_request_context --- localstack/aws/forwarder.py | 5 ++++- tests/unit/aws/protocol/test_parser.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/localstack/aws/forwarder.py b/localstack/aws/forwarder.py index 18737f89edbd9..e2e8383c40c7c 100644 --- a/localstack/aws/forwarder.py +++ b/localstack/aws/forwarder.py @@ -184,8 +184,11 @@ def create_aws_request_context( if auth_path := request_dict.get("auth_path"): # botocore >= 1.28 might modify the url path of the request dict (specifically for S3). # It will then set the original url path as "auth_path". If the auth_path is set, we reset the url_path. + # Since botocore 1.31.2, botocore will strip the query from the `authPart` + # We need to add it back from `requestUri` field # Afterwards the request needs to be prepared again. - request_dict["url_path"] = auth_path + path, sep, query = request_dict["url_path"].partition("?") + request_dict["url_path"] = f"{auth_path}{sep}{query}" prepare_request_dict( request_dict, endpoint_url=endpoint_url, diff --git a/tests/unit/aws/protocol/test_parser.py b/tests/unit/aws/protocol/test_parser.py index 1720c89f9207e..4536cb55f4738 100644 --- a/tests/unit/aws/protocol/test_parser.py +++ b/tests/unit/aws/protocol/test_parser.py @@ -305,8 +305,11 @@ def _botocore_parser_integration_test( # botocore >= 1.28 might modify the url path of the request dict (specifically for S3). # It will then set the original url path as "auth_path". If the auth_path is set, we reset the url_path. + # Since botocore 1.31.2, botocore will strip the query from the `authPart` + # We need to add it back from `requestUri` field if auth_path := serialized_request.get("auth_path"): - serialized_request["url_path"] = auth_path + path, sep, query = serialized_request["url_path"].partition("?") + serialized_request["url_path"] = f"{auth_path}{sep}{query}" prepare_request_dict(serialized_request, "") split_url = urlsplit(serialized_request.get("url"))