From 8715d3d06561190a2f4c04a6f1c451c92c61cf2a Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Thu, 15 May 2025 15:09:08 +0200 Subject: [PATCH] fix IfMatch/IfNoneMatch in pre-signed URLs --- .../localstack/services/s3/presigned_url.py | 13 ++- tests/aws/services/s3/test_s3.py | 82 +++++++++++++++++++ tests/aws/services/s3/test_s3.validation.json | 6 ++ 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/localstack-core/localstack/services/s3/presigned_url.py b/localstack-core/localstack/services/s3/presigned_url.py index ecdd527e65861..e696e82e2c2dc 100644 --- a/localstack-core/localstack/services/s3/presigned_url.py +++ b/localstack-core/localstack/services/s3/presigned_url.py @@ -70,6 +70,14 @@ "x-amz-date", ] +# Boto3 has some issues with some headers that it disregards and does not validate or adds to the signature +# we need to manually define them +# see https://github.com/boto/boto3/issues/4367 +SIGNATURE_V4_BOTO_IGNORED_PARAMS = [ + "if-none-match", + "if-match", +] + # headers to blacklist from request_dict.signed_headers BLACKLISTED_HEADERS = ["X-Amz-Security-Token"] @@ -645,7 +653,10 @@ def _get_signed_headers_and_filtered_query_string( qs_param_low = qs_parameter.lower() if ( qs_parameter not in SIGNATURE_V4_PARAMS - and qs_param_low.startswith("x-amz-") + and ( + qs_param_low.startswith("x-amz-") + or qs_param_low in SIGNATURE_V4_BOTO_IGNORED_PARAMS + ) and qs_param_low not in headers ): if qs_param_low in signed_headers: diff --git a/tests/aws/services/s3/test_s3.py b/tests/aws/services/s3/test_s3.py index 5a0dc6dca7abd..f9e40f87b12c5 100644 --- a/tests/aws/services/s3/test_s3.py +++ b/tests/aws/services/s3/test_s3.py @@ -18,6 +18,7 @@ from zoneinfo import ZoneInfo import boto3 as boto3 +import botocore import pytest import requests import xmltodict @@ -7434,6 +7435,87 @@ def add_content_sha_header(request, **kwargs): resp = requests.put(rewritten_url, data="something", verify=False) assert resp.status_code == 403 + @markers.aws.validated + def test_pre_signed_url_if_none_match(self, s3_bucket, aws_client, aws_session): + # there currently is a bug in Boto3: https://github.com/boto/boto3/issues/4367 + # so we need to use botocore directly to allow testing of this, as other SDK like the Java SDK have the correct + # behavior + object_key = "temp.txt" + + s3_endpoint_path_style = _endpoint_url() + + # assert that the regular Boto3 client does not work, and does not sign the parameter as requested + client = _s3_client_pre_signed_client( + Config(signature_version="s3v4", s3={}), + endpoint_url=s3_endpoint_path_style, + ) + bad_url = client.generate_presigned_url( + "put_object", + Params={"Bucket": s3_bucket, "Key": object_key, "IfNoneMatch": "*"}, + ) + assert "if-none-match=%2a" not in bad_url.lower() + + req = botocore.awsrequest.AWSRequest( + method="PUT", + url=f"{s3_endpoint_path_style}/{s3_bucket}/{object_key}", + data={}, + params={ + "If-None-Match": "*", + }, + headers={}, + ) + + botocore.auth.S3SigV4QueryAuth(aws_session.get_credentials(), "s3", "us-east-1").add_auth( + req + ) + + assert "if-none-match=%2a" in req.url.lower() + + response = requests.put(req.url) + assert response.status_code == 200 + + response = requests.put(req.url) + # we are now failing because the object already exists + assert response.status_code == 412 + + @markers.aws.validated + def test_pre_signed_url_if_match(self, s3_bucket, aws_client, aws_session): + key = "test-precondition" + aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test") + + s3_endpoint_path_style = _endpoint_url() + # empty object ETag is provided + empty_object_etag = "d41d8cd98f00b204e9800998ecf8427e" + + # assert that the regular Boto3 client does not work, and does not sign the parameter as requested + client = _s3_client_pre_signed_client( + Config(signature_version="s3v4", s3={}), + endpoint_url=s3_endpoint_path_style, + ) + bad_url = client.generate_presigned_url( + "put_object", + Params={"Bucket": s3_bucket, "Key": key, "IfMatch": empty_object_etag}, + ) + assert "if-match=d41d8cd98f00b204e9800998ecf8427e" not in bad_url.lower() + + req = botocore.awsrequest.AWSRequest( + method="PUT", + url=f"{s3_endpoint_path_style}/{s3_bucket}/{key}", + data={}, + params={ + "If-Match": empty_object_etag, + }, + headers={}, + ) + + botocore.auth.S3SigV4QueryAuth(aws_session.get_credentials(), "s3", "us-east-1").add_auth( + req + ) + assert "if-match=d41d8cd98f00b204e9800998ecf8427e" in req.url.lower() + + response = requests.put(req.url) + assert response.status_code == 412 + class TestS3DeepArchive: """ diff --git a/tests/aws/services/s3/test_s3.validation.json b/tests/aws/services/s3/test_s3.validation.json index d26a2cce6ff21..dcc4ca26324c6 100644 --- a/tests/aws/services/s3/test_s3.validation.json +++ b/tests/aws/services/s3/test_s3.validation.json @@ -788,6 +788,12 @@ "tests/aws/services/s3/test_s3.py::TestS3PresignedUrl::test_pre_signed_url_forward_slash_bucket": { "last_validated_date": "2025-01-21T18:25:38+00:00" }, + "tests/aws/services/s3/test_s3.py::TestS3PresignedUrl::test_pre_signed_url_if_match": { + "last_validated_date": "2025-05-15T13:08:44+00:00" + }, + "tests/aws/services/s3/test_s3.py::TestS3PresignedUrl::test_pre_signed_url_if_none_match": { + "last_validated_date": "2025-05-15T12:51:09+00:00" + }, "tests/aws/services/s3/test_s3.py::TestS3PresignedUrl::test_presign_with_additional_query_params": { "last_validated_date": "2025-01-21T18:22:43+00:00" },